1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 93 { 94 int fc_type = 0; 95 switch (type) { 96 case PASSTHROUGH: 97 fc_type = FC_GEN; 98 break; 99 case DES3_CBC: 100 case DES3_ECB: 101 fc_type = FC_GEN; 102 break; 103 case AES_CBC: 104 case AES_ECB: 105 case AES_CFB: 106 case AES_CTR: 107 case AES_GCM: 108 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 109 return -1; 110 fc_type = FC_GEN; 111 break; 112 case AES_XTS: 113 key_len = key_len / 2; 114 if (unlikely(key_len == CPT_BYTE_24)) { 115 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 116 return -1; 117 } 118 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 119 return -1; 120 fc_type = FC_GEN; 121 break; 122 case ZUC_EEA3: 123 case SNOW3G_UEA2: 124 if (unlikely(key_len != 16)) 125 return -1; 126 /* No support for AEAD yet */ 127 if (unlikely(ctx->hash_type)) 128 return -1; 129 fc_type = ZUC_SNOW3G; 130 break; 131 case KASUMI_F8_CBC: 132 case KASUMI_F8_ECB: 133 if (unlikely(key_len != 16)) 134 return -1; 135 /* No support for AEAD yet */ 136 if (unlikely(ctx->hash_type)) 137 return -1; 138 fc_type = KASUMI; 139 break; 140 default: 141 return -1; 142 } 143 144 ctx->fc_type = fc_type; 145 return 0; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 150 { 151 cpt_ctx->enc_cipher = 0; 152 fctx->enc.enc_cipher = 0; 153 } 154 155 static __rte_always_inline void 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 157 { 158 mc_aes_type_t aes_key_type = 0; 159 switch (key_len) { 160 case CPT_BYTE_16: 161 aes_key_type = AES_128_BIT; 162 break; 163 case CPT_BYTE_24: 164 aes_key_type = AES_192_BIT; 165 break; 166 case CPT_BYTE_32: 167 aes_key_type = AES_256_BIT; 168 break; 169 default: 170 /* This should not happen */ 171 CPT_LOG_DP_ERR("Invalid AES key len"); 172 return; 173 } 174 fctx->enc.aes_key = aes_key_type; 175 } 176 177 static __rte_always_inline void 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 179 uint16_t key_len) 180 { 181 uint32_t keyx[4]; 182 cpt_ctx->snow3g = 1; 183 gen_key_snow3g(key, keyx); 184 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->zsk_flags = 0; 196 } 197 198 static __rte_always_inline void 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 200 uint16_t key_len) 201 { 202 cpt_ctx->k_ecb = 1; 203 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 204 cpt_ctx->zsk_flags = 0; 205 } 206 207 static __rte_always_inline void 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 209 uint16_t key_len) 210 { 211 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 212 cpt_ctx->zsk_flags = 0; 213 } 214 215 static __rte_always_inline int 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 217 uint16_t key_len, uint8_t *salt) 218 { 219 struct cpt_ctx *cpt_ctx = ctx; 220 mc_fc_context_t *fctx = &cpt_ctx->fctx; 221 int ret; 222 223 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 224 if (unlikely(ret)) 225 return -1; 226 227 if (cpt_ctx->fc_type == FC_GEN) { 228 /* 229 * We need to always say IV is from DPTR as user can 230 * sometimes iverride IV per operation. 231 */ 232 fctx->enc.iv_source = CPT_FROM_DPTR; 233 234 if (cpt_ctx->auth_key_len > 64) 235 return -1; 236 } 237 238 switch (type) { 239 case PASSTHROUGH: 240 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 241 goto success; 242 case DES3_CBC: 243 /* CPT performs DES using 3DES with the 8B DES-key 244 * replicated 2 more times to match the 24B 3DES-key. 245 * Eg. If org. key is "0x0a 0x0b", then new key is 246 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 247 */ 248 if (key_len == 8) { 249 /* Skipping the first 8B as it will be copied 250 * in the regular code flow 251 */ 252 memcpy(fctx->enc.encr_key+key_len, key, key_len); 253 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 254 } 255 break; 256 case DES3_ECB: 257 /* For DES3_ECB IV need to be from CTX. */ 258 fctx->enc.iv_source = CPT_FROM_CTX; 259 break; 260 case AES_CBC: 261 case AES_ECB: 262 case AES_CFB: 263 case AES_CTR: 264 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 265 break; 266 case AES_GCM: 267 /* Even though iv source is from dptr, 268 * aes_gcm salt is taken from ctx 269 */ 270 if (salt) { 271 memcpy(fctx->enc.encr_iv, salt, 4); 272 /* Assuming it was just salt update 273 * and nothing else 274 */ 275 if (!key) 276 goto success; 277 } 278 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 279 break; 280 case AES_XTS: 281 key_len = key_len / 2; 282 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 283 284 /* Copy key2 for XTS into ipad */ 285 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 286 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 287 break; 288 case SNOW3G_UEA2: 289 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 290 goto success; 291 case ZUC_EEA3: 292 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 293 goto success; 294 case KASUMI_F8_ECB: 295 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 296 goto success; 297 case KASUMI_F8_CBC: 298 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 299 goto success; 300 default: 301 break; 302 } 303 304 /* Only for FC_GEN case */ 305 306 /* For GMAC auth, cipher must be NULL */ 307 if (cpt_ctx->hash_type != GMAC_TYPE) 308 fctx->enc.enc_cipher = type; 309 310 memcpy(fctx->enc.encr_key, key, key_len); 311 312 success: 313 cpt_ctx->enc_cipher = type; 314 315 return 0; 316 } 317 318 static __rte_always_inline uint32_t 319 fill_sg_comp(sg_comp_t *list, 320 uint32_t i, 321 phys_addr_t dma_addr, 322 uint32_t size) 323 { 324 sg_comp_t *to = &list[i>>2]; 325 326 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 327 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 328 i++; 329 return i; 330 } 331 332 static __rte_always_inline uint32_t 333 fill_sg_comp_from_buf(sg_comp_t *list, 334 uint32_t i, 335 buf_ptr_t *from) 336 { 337 sg_comp_t *to = &list[i>>2]; 338 339 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 340 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 341 i++; 342 return i; 343 } 344 345 static __rte_always_inline uint32_t 346 fill_sg_comp_from_buf_min(sg_comp_t *list, 347 uint32_t i, 348 buf_ptr_t *from, 349 uint32_t *psize) 350 { 351 sg_comp_t *to = &list[i >> 2]; 352 uint32_t size = *psize; 353 uint32_t e_len; 354 355 e_len = (size > from->size) ? from->size : size; 356 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 357 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 358 *psize -= e_len; 359 i++; 360 return i; 361 } 362 363 /* 364 * This fills the MC expected SGIO list 365 * from IOV given by user. 366 */ 367 static __rte_always_inline uint32_t 368 fill_sg_comp_from_iov(sg_comp_t *list, 369 uint32_t i, 370 iov_ptr_t *from, uint32_t from_offset, 371 uint32_t *psize, buf_ptr_t *extra_buf, 372 uint32_t extra_offset) 373 { 374 int32_t j; 375 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 376 uint32_t size = *psize - extra_len; 377 buf_ptr_t *bufs; 378 379 bufs = from->bufs; 380 for (j = 0; (j < from->buf_cnt) && size; j++) { 381 phys_addr_t e_dma_addr; 382 uint32_t e_len; 383 sg_comp_t *to = &list[i >> 2]; 384 385 if (!bufs[j].size) 386 continue; 387 388 if (unlikely(from_offset)) { 389 if (from_offset >= bufs[j].size) { 390 from_offset -= bufs[j].size; 391 continue; 392 } 393 e_dma_addr = bufs[j].dma_addr + from_offset; 394 e_len = (size > (bufs[j].size - from_offset)) ? 395 (bufs[j].size - from_offset) : size; 396 from_offset = 0; 397 } else { 398 e_dma_addr = bufs[j].dma_addr; 399 e_len = (size > bufs[j].size) ? 400 bufs[j].size : size; 401 } 402 403 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 404 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 405 406 if (extra_len && (e_len >= extra_offset)) { 407 /* Break the data at given offset */ 408 uint32_t next_len = e_len - extra_offset; 409 phys_addr_t next_dma = e_dma_addr + extra_offset; 410 411 if (!extra_offset) { 412 i--; 413 } else { 414 e_len = extra_offset; 415 size -= e_len; 416 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 417 } 418 419 /* Insert extra data ptr */ 420 if (extra_len) { 421 i++; 422 to = &list[i >> 2]; 423 to->u.s.len[i % 4] = 424 rte_cpu_to_be_16(extra_buf->size); 425 to->ptr[i % 4] = 426 rte_cpu_to_be_64(extra_buf->dma_addr); 427 428 /* size already decremented by extra len */ 429 } 430 431 /* insert the rest of the data */ 432 if (next_len) { 433 i++; 434 to = &list[i >> 2]; 435 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 436 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 437 size -= next_len; 438 } 439 extra_len = 0; 440 441 } else { 442 size -= e_len; 443 } 444 if (extra_offset) 445 extra_offset -= size; 446 i++; 447 } 448 449 *psize = size; 450 return (uint32_t)i; 451 } 452 453 static __rte_always_inline void 454 cpt_digest_gen_prep(uint32_t flags, 455 uint64_t d_lens, 456 digest_params_t *params, 457 void *op, 458 void **prep_req) 459 { 460 struct cpt_request_info *req; 461 uint32_t size, i; 462 uint16_t data_len, mac_len, key_len; 463 auth_type_t hash_type; 464 buf_ptr_t *meta_p; 465 struct cpt_ctx *ctx; 466 sg_comp_t *gather_comp; 467 sg_comp_t *scatter_comp; 468 uint8_t *in_buffer; 469 uint32_t g_size_bytes, s_size_bytes; 470 uint64_t dptr_dma, rptr_dma; 471 vq_cmd_word0_t vq_cmd_w0; 472 vq_cmd_word3_t vq_cmd_w3; 473 void *c_vaddr, *m_vaddr; 474 uint64_t c_dma, m_dma; 475 opcode_info_t opcode; 476 477 ctx = params->ctx_buf.vaddr; 478 meta_p = ¶ms->meta_buf; 479 480 m_vaddr = meta_p->vaddr; 481 m_dma = meta_p->dma_addr; 482 483 /* 484 * Save initial space that followed app data for completion code & 485 * alternate completion code to fall in same cache line as app data 486 */ 487 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 488 m_dma += COMPLETION_CODE_SIZE; 489 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 490 (uint8_t *)m_vaddr; 491 c_vaddr = (uint8_t *)m_vaddr + size; 492 c_dma = m_dma + size; 493 size += sizeof(cpt_res_s_t); 494 495 m_vaddr = (uint8_t *)m_vaddr + size; 496 m_dma += size; 497 498 req = m_vaddr; 499 500 size = sizeof(struct cpt_request_info); 501 m_vaddr = (uint8_t *)m_vaddr + size; 502 m_dma += size; 503 504 hash_type = ctx->hash_type; 505 mac_len = ctx->mac_len; 506 key_len = ctx->auth_key_len; 507 data_len = AUTH_DLEN(d_lens); 508 509 /*GP op header */ 510 vq_cmd_w0.u64 = 0; 511 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 512 if (ctx->hmac) { 513 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 514 vq_cmd_w0.s.param1 = key_len; 515 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 516 } else { 517 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 518 vq_cmd_w0.s.param1 = 0; 519 vq_cmd_w0.s.dlen = data_len; 520 } 521 522 opcode.s.minor = 0; 523 524 /* Null auth only case enters the if */ 525 if (unlikely(!hash_type && !ctx->enc_cipher)) { 526 opcode.s.major = CPT_MAJOR_OP_MISC; 527 /* Minor op is passthrough */ 528 opcode.s.minor = 0x03; 529 /* Send out completion code only */ 530 vq_cmd_w0.s.param2 = 0x1; 531 } 532 533 vq_cmd_w0.s.opcode = opcode.flags; 534 535 /* DPTR has SG list */ 536 in_buffer = m_vaddr; 537 dptr_dma = m_dma; 538 539 ((uint16_t *)in_buffer)[0] = 0; 540 ((uint16_t *)in_buffer)[1] = 0; 541 542 /* TODO Add error check if space will be sufficient */ 543 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 544 545 /* 546 * Input gather list 547 */ 548 549 i = 0; 550 551 if (ctx->hmac) { 552 uint64_t k_dma = params->ctx_buf.dma_addr + 553 offsetof(struct cpt_ctx, auth_key); 554 /* Key */ 555 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 556 } 557 558 /* input data */ 559 size = data_len; 560 if (size) { 561 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 562 0, &size, NULL, 0); 563 if (unlikely(size)) { 564 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 565 " by %dB", size); 566 return; 567 } 568 } else { 569 /* 570 * Looks like we need to support zero data 571 * gather ptr in case of hash & hmac 572 */ 573 i++; 574 } 575 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 576 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 577 578 /* 579 * Output Gather list 580 */ 581 582 i = 0; 583 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 584 585 if (flags & VALID_MAC_BUF) { 586 if (unlikely(params->mac_buf.size < mac_len)) { 587 CPT_LOG_DP_ERR("Insufficient MAC size"); 588 return; 589 } 590 591 size = mac_len; 592 i = fill_sg_comp_from_buf_min(scatter_comp, i, 593 ¶ms->mac_buf, &size); 594 } else { 595 size = mac_len; 596 i = fill_sg_comp_from_iov(scatter_comp, i, 597 params->src_iov, data_len, 598 &size, NULL, 0); 599 if (unlikely(size)) { 600 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 601 " %dB", size); 602 return; 603 } 604 } 605 606 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 607 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 608 609 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 610 611 /* This is DPTR len incase of SG mode */ 612 vq_cmd_w0.s.dlen = size; 613 614 m_vaddr = (uint8_t *)m_vaddr + size; 615 m_dma += size; 616 617 /* cpt alternate completion address saved earlier */ 618 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 619 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 620 rptr_dma = c_dma - 8; 621 622 req->ist.ei1 = dptr_dma; 623 req->ist.ei2 = rptr_dma; 624 625 /* vq command w3 */ 626 vq_cmd_w3.u64 = 0; 627 628 /* 16 byte aligned cpt res address */ 629 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 630 *req->completion_addr = COMPLETION_CODE_INIT; 631 req->comp_baddr = c_dma; 632 633 /* Fill microcode part of instruction */ 634 req->ist.ei0 = vq_cmd_w0.u64; 635 req->ist.ei3 = vq_cmd_w3.u64; 636 637 req->op = op; 638 639 *prep_req = req; 640 return; 641 } 642 643 static __rte_always_inline void 644 cpt_enc_hmac_prep(uint32_t flags, 645 uint64_t d_offs, 646 uint64_t d_lens, 647 fc_params_t *fc_params, 648 void *op, 649 void **prep_req) 650 { 651 uint32_t iv_offset = 0; 652 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 653 struct cpt_ctx *cpt_ctx; 654 uint32_t cipher_type, hash_type; 655 uint32_t mac_len, size; 656 uint8_t iv_len = 16; 657 struct cpt_request_info *req; 658 buf_ptr_t *meta_p, *aad_buf = NULL; 659 uint32_t encr_offset, auth_offset; 660 uint32_t encr_data_len, auth_data_len, aad_len = 0; 661 uint32_t passthrough_len = 0; 662 void *m_vaddr, *offset_vaddr; 663 uint64_t m_dma, offset_dma, ctx_dma; 664 vq_cmd_word0_t vq_cmd_w0; 665 vq_cmd_word3_t vq_cmd_w3; 666 void *c_vaddr; 667 uint64_t c_dma; 668 opcode_info_t opcode; 669 670 meta_p = &fc_params->meta_buf; 671 m_vaddr = meta_p->vaddr; 672 m_dma = meta_p->dma_addr; 673 674 encr_offset = ENCR_OFFSET(d_offs); 675 auth_offset = AUTH_OFFSET(d_offs); 676 encr_data_len = ENCR_DLEN(d_lens); 677 auth_data_len = AUTH_DLEN(d_lens); 678 if (unlikely(flags & VALID_AAD_BUF)) { 679 /* 680 * We dont support both aad 681 * and auth data separately 682 */ 683 auth_data_len = 0; 684 auth_offset = 0; 685 aad_len = fc_params->aad_buf.size; 686 aad_buf = &fc_params->aad_buf; 687 } 688 cpt_ctx = fc_params->ctx_buf.vaddr; 689 cipher_type = cpt_ctx->enc_cipher; 690 hash_type = cpt_ctx->hash_type; 691 mac_len = cpt_ctx->mac_len; 692 693 /* 694 * Save initial space that followed app data for completion code & 695 * alternate completion code to fall in same cache line as app data 696 */ 697 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 698 m_dma += COMPLETION_CODE_SIZE; 699 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 700 (uint8_t *)m_vaddr; 701 702 c_vaddr = (uint8_t *)m_vaddr + size; 703 c_dma = m_dma + size; 704 size += sizeof(cpt_res_s_t); 705 706 m_vaddr = (uint8_t *)m_vaddr + size; 707 m_dma += size; 708 709 /* start cpt request info struct at 8 byte boundary */ 710 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 711 (uint8_t *)m_vaddr; 712 713 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 714 715 size += sizeof(struct cpt_request_info); 716 m_vaddr = (uint8_t *)m_vaddr + size; 717 m_dma += size; 718 719 if (hash_type == GMAC_TYPE) 720 encr_data_len = 0; 721 722 if (unlikely(!(flags & VALID_IV_BUF))) { 723 iv_len = 0; 724 iv_offset = ENCR_IV_OFFSET(d_offs); 725 } 726 727 if (unlikely(flags & VALID_AAD_BUF)) { 728 /* 729 * When AAD is given, data above encr_offset is pass through 730 * Since AAD is given as separate pointer and not as offset, 731 * this is a special case as we need to fragment input data 732 * into passthrough + encr_data and then insert AAD in between. 733 */ 734 if (hash_type != GMAC_TYPE) { 735 passthrough_len = encr_offset; 736 auth_offset = passthrough_len + iv_len; 737 encr_offset = passthrough_len + aad_len + iv_len; 738 auth_data_len = aad_len + encr_data_len; 739 } else { 740 passthrough_len = 16 + aad_len; 741 auth_offset = passthrough_len + iv_len; 742 auth_data_len = aad_len; 743 } 744 } else { 745 encr_offset += iv_len; 746 auth_offset += iv_len; 747 } 748 749 /* Encryption */ 750 opcode.s.major = CPT_MAJOR_OP_FC; 751 opcode.s.minor = 0; 752 753 auth_dlen = auth_offset + auth_data_len; 754 enc_dlen = encr_data_len + encr_offset; 755 if (unlikely(encr_data_len & 0xf)) { 756 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 757 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 758 else if (likely((cipher_type == AES_CBC) || 759 (cipher_type == AES_ECB))) 760 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 761 } 762 763 if (unlikely(hash_type == GMAC_TYPE)) { 764 encr_offset = auth_dlen; 765 enc_dlen = 0; 766 } 767 768 if (unlikely(auth_dlen > enc_dlen)) { 769 inputlen = auth_dlen; 770 outputlen = auth_dlen + mac_len; 771 } else { 772 inputlen = enc_dlen; 773 outputlen = enc_dlen + mac_len; 774 } 775 776 /* GP op header */ 777 vq_cmd_w0.u64 = 0; 778 vq_cmd_w0.s.param1 = encr_data_len; 779 vq_cmd_w0.s.param2 = auth_data_len; 780 /* 781 * In 83XX since we have a limitation of 782 * IV & Offset control word not part of instruction 783 * and need to be part of Data Buffer, we check if 784 * head room is there and then only do the Direct mode processing 785 */ 786 if (likely((flags & SINGLE_BUF_INPLACE) && 787 (flags & SINGLE_BUF_HEADTAILROOM))) { 788 void *dm_vaddr = fc_params->bufs[0].vaddr; 789 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 790 /* 791 * This flag indicates that there is 24 bytes head room and 792 * 8 bytes tail room available, so that we get to do 793 * DIRECT MODE with limitation 794 */ 795 796 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 797 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 798 799 /* DPTR */ 800 req->ist.ei1 = offset_dma; 801 /* RPTR should just exclude offset control word */ 802 req->ist.ei2 = dm_dma_addr - iv_len; 803 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 804 + outputlen - iv_len); 805 806 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 807 808 vq_cmd_w0.s.opcode = opcode.flags; 809 810 if (likely(iv_len)) { 811 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 812 + OFF_CTRL_LEN); 813 uint64_t *src = fc_params->iv_buf; 814 dest[0] = src[0]; 815 dest[1] = src[1]; 816 } 817 818 *(uint64_t *)offset_vaddr = 819 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 820 ((uint64_t)iv_offset << 8) | 821 ((uint64_t)auth_offset)); 822 823 } else { 824 uint32_t i, g_size_bytes, s_size_bytes; 825 uint64_t dptr_dma, rptr_dma; 826 sg_comp_t *gather_comp; 827 sg_comp_t *scatter_comp; 828 uint8_t *in_buffer; 829 830 /* This falls under strict SG mode */ 831 offset_vaddr = m_vaddr; 832 offset_dma = m_dma; 833 size = OFF_CTRL_LEN + iv_len; 834 835 m_vaddr = (uint8_t *)m_vaddr + size; 836 m_dma += size; 837 838 opcode.s.major |= CPT_DMA_MODE; 839 840 vq_cmd_w0.s.opcode = opcode.flags; 841 842 if (likely(iv_len)) { 843 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 844 + OFF_CTRL_LEN); 845 uint64_t *src = fc_params->iv_buf; 846 dest[0] = src[0]; 847 dest[1] = src[1]; 848 } 849 850 *(uint64_t *)offset_vaddr = 851 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 852 ((uint64_t)iv_offset << 8) | 853 ((uint64_t)auth_offset)); 854 855 /* DPTR has SG list */ 856 in_buffer = m_vaddr; 857 dptr_dma = m_dma; 858 859 ((uint16_t *)in_buffer)[0] = 0; 860 ((uint16_t *)in_buffer)[1] = 0; 861 862 /* TODO Add error check if space will be sufficient */ 863 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 864 865 /* 866 * Input Gather List 867 */ 868 869 i = 0; 870 871 /* Offset control word that includes iv */ 872 i = fill_sg_comp(gather_comp, i, offset_dma, 873 OFF_CTRL_LEN + iv_len); 874 875 /* Add input data */ 876 size = inputlen - iv_len; 877 if (likely(size)) { 878 uint32_t aad_offset = aad_len ? passthrough_len : 0; 879 880 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 881 i = fill_sg_comp_from_buf_min(gather_comp, i, 882 fc_params->bufs, 883 &size); 884 } else { 885 i = fill_sg_comp_from_iov(gather_comp, i, 886 fc_params->src_iov, 887 0, &size, 888 aad_buf, aad_offset); 889 } 890 891 if (unlikely(size)) { 892 CPT_LOG_DP_ERR("Insufficient buffer space," 893 " size %d needed", size); 894 return; 895 } 896 } 897 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 898 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 899 900 /* 901 * Output Scatter list 902 */ 903 i = 0; 904 scatter_comp = 905 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 906 907 /* Add IV */ 908 if (likely(iv_len)) { 909 i = fill_sg_comp(scatter_comp, i, 910 offset_dma + OFF_CTRL_LEN, 911 iv_len); 912 } 913 914 /* output data or output data + digest*/ 915 if (unlikely(flags & VALID_MAC_BUF)) { 916 size = outputlen - iv_len - mac_len; 917 if (size) { 918 uint32_t aad_offset = 919 aad_len ? passthrough_len : 0; 920 921 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 922 i = fill_sg_comp_from_buf_min( 923 scatter_comp, 924 i, 925 fc_params->bufs, 926 &size); 927 } else { 928 i = fill_sg_comp_from_iov(scatter_comp, 929 i, 930 fc_params->dst_iov, 931 0, 932 &size, 933 aad_buf, 934 aad_offset); 935 } 936 if (unlikely(size)) { 937 CPT_LOG_DP_ERR("Insufficient buffer" 938 " space, size %d needed", 939 size); 940 return; 941 } 942 } 943 /* mac_data */ 944 if (mac_len) { 945 i = fill_sg_comp_from_buf(scatter_comp, i, 946 &fc_params->mac_buf); 947 } 948 } else { 949 /* Output including mac */ 950 size = outputlen - iv_len; 951 if (likely(size)) { 952 uint32_t aad_offset = 953 aad_len ? passthrough_len : 0; 954 955 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 956 i = fill_sg_comp_from_buf_min( 957 scatter_comp, 958 i, 959 fc_params->bufs, 960 &size); 961 } else { 962 i = fill_sg_comp_from_iov(scatter_comp, 963 i, 964 fc_params->dst_iov, 965 0, 966 &size, 967 aad_buf, 968 aad_offset); 969 } 970 if (unlikely(size)) { 971 CPT_LOG_DP_ERR("Insufficient buffer" 972 " space, size %d needed", 973 size); 974 return; 975 } 976 } 977 } 978 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 979 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 980 981 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 982 983 /* This is DPTR len incase of SG mode */ 984 vq_cmd_w0.s.dlen = size; 985 986 m_vaddr = (uint8_t *)m_vaddr + size; 987 m_dma += size; 988 989 /* cpt alternate completion address saved earlier */ 990 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 991 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 992 rptr_dma = c_dma - 8; 993 994 req->ist.ei1 = dptr_dma; 995 req->ist.ei2 = rptr_dma; 996 } 997 998 ctx_dma = fc_params->ctx_buf.dma_addr + 999 offsetof(struct cpt_ctx, fctx); 1000 /* vq command w3 */ 1001 vq_cmd_w3.u64 = 0; 1002 vq_cmd_w3.s.grp = 0; 1003 vq_cmd_w3.s.cptr = ctx_dma; 1004 1005 /* 16 byte aligned cpt res address */ 1006 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1007 *req->completion_addr = COMPLETION_CODE_INIT; 1008 req->comp_baddr = c_dma; 1009 1010 /* Fill microcode part of instruction */ 1011 req->ist.ei0 = vq_cmd_w0.u64; 1012 req->ist.ei3 = vq_cmd_w3.u64; 1013 1014 req->op = op; 1015 1016 *prep_req = req; 1017 return; 1018 } 1019 1020 static __rte_always_inline void 1021 cpt_dec_hmac_prep(uint32_t flags, 1022 uint64_t d_offs, 1023 uint64_t d_lens, 1024 fc_params_t *fc_params, 1025 void *op, 1026 void **prep_req) 1027 { 1028 uint32_t iv_offset = 0, size; 1029 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1030 struct cpt_ctx *cpt_ctx; 1031 int32_t hash_type, mac_len; 1032 uint8_t iv_len = 16; 1033 struct cpt_request_info *req; 1034 buf_ptr_t *meta_p, *aad_buf = NULL; 1035 uint32_t encr_offset, auth_offset; 1036 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1037 uint32_t passthrough_len = 0; 1038 void *m_vaddr, *offset_vaddr; 1039 uint64_t m_dma, offset_dma, ctx_dma; 1040 opcode_info_t opcode; 1041 vq_cmd_word0_t vq_cmd_w0; 1042 vq_cmd_word3_t vq_cmd_w3; 1043 void *c_vaddr; 1044 uint64_t c_dma; 1045 1046 meta_p = &fc_params->meta_buf; 1047 m_vaddr = meta_p->vaddr; 1048 m_dma = meta_p->dma_addr; 1049 1050 encr_offset = ENCR_OFFSET(d_offs); 1051 auth_offset = AUTH_OFFSET(d_offs); 1052 encr_data_len = ENCR_DLEN(d_lens); 1053 auth_data_len = AUTH_DLEN(d_lens); 1054 1055 if (unlikely(flags & VALID_AAD_BUF)) { 1056 /* 1057 * We dont support both aad 1058 * and auth data separately 1059 */ 1060 auth_data_len = 0; 1061 auth_offset = 0; 1062 aad_len = fc_params->aad_buf.size; 1063 aad_buf = &fc_params->aad_buf; 1064 } 1065 1066 cpt_ctx = fc_params->ctx_buf.vaddr; 1067 hash_type = cpt_ctx->hash_type; 1068 mac_len = cpt_ctx->mac_len; 1069 1070 if (hash_type == GMAC_TYPE) 1071 encr_data_len = 0; 1072 1073 if (unlikely(!(flags & VALID_IV_BUF))) { 1074 iv_len = 0; 1075 iv_offset = ENCR_IV_OFFSET(d_offs); 1076 } 1077 1078 if (unlikely(flags & VALID_AAD_BUF)) { 1079 /* 1080 * When AAD is given, data above encr_offset is pass through 1081 * Since AAD is given as separate pointer and not as offset, 1082 * this is a special case as we need to fragment input data 1083 * into passthrough + encr_data and then insert AAD in between. 1084 */ 1085 if (hash_type != GMAC_TYPE) { 1086 passthrough_len = encr_offset; 1087 auth_offset = passthrough_len + iv_len; 1088 encr_offset = passthrough_len + aad_len + iv_len; 1089 auth_data_len = aad_len + encr_data_len; 1090 } else { 1091 passthrough_len = 16 + aad_len; 1092 auth_offset = passthrough_len + iv_len; 1093 auth_data_len = aad_len; 1094 } 1095 } else { 1096 encr_offset += iv_len; 1097 auth_offset += iv_len; 1098 } 1099 1100 /* 1101 * Save initial space that followed app data for completion code & 1102 * alternate completion code to fall in same cache line as app data 1103 */ 1104 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1105 m_dma += COMPLETION_CODE_SIZE; 1106 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1107 (uint8_t *)m_vaddr; 1108 c_vaddr = (uint8_t *)m_vaddr + size; 1109 c_dma = m_dma + size; 1110 size += sizeof(cpt_res_s_t); 1111 1112 m_vaddr = (uint8_t *)m_vaddr + size; 1113 m_dma += size; 1114 1115 /* start cpt request info structure at 8 byte alignment */ 1116 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1117 (uint8_t *)m_vaddr; 1118 1119 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1120 1121 size += sizeof(struct cpt_request_info); 1122 m_vaddr = (uint8_t *)m_vaddr + size; 1123 m_dma += size; 1124 1125 /* Decryption */ 1126 opcode.s.major = CPT_MAJOR_OP_FC; 1127 opcode.s.minor = 1; 1128 1129 enc_dlen = encr_offset + encr_data_len; 1130 auth_dlen = auth_offset + auth_data_len; 1131 1132 if (auth_dlen > enc_dlen) { 1133 inputlen = auth_dlen + mac_len; 1134 outputlen = auth_dlen; 1135 } else { 1136 inputlen = enc_dlen + mac_len; 1137 outputlen = enc_dlen; 1138 } 1139 1140 if (hash_type == GMAC_TYPE) 1141 encr_offset = inputlen; 1142 1143 vq_cmd_w0.u64 = 0; 1144 vq_cmd_w0.s.param1 = encr_data_len; 1145 vq_cmd_w0.s.param2 = auth_data_len; 1146 1147 /* 1148 * In 83XX since we have a limitation of 1149 * IV & Offset control word not part of instruction 1150 * and need to be part of Data Buffer, we check if 1151 * head room is there and then only do the Direct mode processing 1152 */ 1153 if (likely((flags & SINGLE_BUF_INPLACE) && 1154 (flags & SINGLE_BUF_HEADTAILROOM))) { 1155 void *dm_vaddr = fc_params->bufs[0].vaddr; 1156 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1157 /* 1158 * This flag indicates that there is 24 bytes head room and 1159 * 8 bytes tail room available, so that we get to do 1160 * DIRECT MODE with limitation 1161 */ 1162 1163 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1164 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1165 req->ist.ei1 = offset_dma; 1166 1167 /* RPTR should just exclude offset control word */ 1168 req->ist.ei2 = dm_dma_addr - iv_len; 1169 1170 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1171 outputlen - iv_len); 1172 /* since this is decryption, 1173 * don't touch the content of 1174 * alternate ccode space as it contains 1175 * hmac. 1176 */ 1177 1178 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1179 1180 vq_cmd_w0.s.opcode = opcode.flags; 1181 1182 if (likely(iv_len)) { 1183 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1184 OFF_CTRL_LEN); 1185 uint64_t *src = fc_params->iv_buf; 1186 dest[0] = src[0]; 1187 dest[1] = src[1]; 1188 } 1189 1190 *(uint64_t *)offset_vaddr = 1191 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1192 ((uint64_t)iv_offset << 8) | 1193 ((uint64_t)auth_offset)); 1194 1195 } else { 1196 uint64_t dptr_dma, rptr_dma; 1197 uint32_t g_size_bytes, s_size_bytes; 1198 sg_comp_t *gather_comp; 1199 sg_comp_t *scatter_comp; 1200 uint8_t *in_buffer; 1201 uint8_t i = 0; 1202 1203 /* This falls under strict SG mode */ 1204 offset_vaddr = m_vaddr; 1205 offset_dma = m_dma; 1206 size = OFF_CTRL_LEN + iv_len; 1207 1208 m_vaddr = (uint8_t *)m_vaddr + size; 1209 m_dma += size; 1210 1211 opcode.s.major |= CPT_DMA_MODE; 1212 1213 vq_cmd_w0.s.opcode = opcode.flags; 1214 1215 if (likely(iv_len)) { 1216 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1217 OFF_CTRL_LEN); 1218 uint64_t *src = fc_params->iv_buf; 1219 dest[0] = src[0]; 1220 dest[1] = src[1]; 1221 } 1222 1223 *(uint64_t *)offset_vaddr = 1224 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1225 ((uint64_t)iv_offset << 8) | 1226 ((uint64_t)auth_offset)); 1227 1228 /* DPTR has SG list */ 1229 in_buffer = m_vaddr; 1230 dptr_dma = m_dma; 1231 1232 ((uint16_t *)in_buffer)[0] = 0; 1233 ((uint16_t *)in_buffer)[1] = 0; 1234 1235 /* TODO Add error check if space will be sufficient */ 1236 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1237 1238 /* 1239 * Input Gather List 1240 */ 1241 i = 0; 1242 1243 /* Offset control word that includes iv */ 1244 i = fill_sg_comp(gather_comp, i, offset_dma, 1245 OFF_CTRL_LEN + iv_len); 1246 1247 /* Add input data */ 1248 if (flags & VALID_MAC_BUF) { 1249 size = inputlen - iv_len - mac_len; 1250 if (size) { 1251 /* input data only */ 1252 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1253 i = fill_sg_comp_from_buf_min( 1254 gather_comp, i, 1255 fc_params->bufs, 1256 &size); 1257 } else { 1258 uint32_t aad_offset = aad_len ? 1259 passthrough_len : 0; 1260 1261 i = fill_sg_comp_from_iov(gather_comp, 1262 i, 1263 fc_params->src_iov, 1264 0, &size, 1265 aad_buf, 1266 aad_offset); 1267 } 1268 if (unlikely(size)) { 1269 CPT_LOG_DP_ERR("Insufficient buffer" 1270 " space, size %d needed", 1271 size); 1272 return; 1273 } 1274 } 1275 1276 /* mac data */ 1277 if (mac_len) { 1278 i = fill_sg_comp_from_buf(gather_comp, i, 1279 &fc_params->mac_buf); 1280 } 1281 } else { 1282 /* input data + mac */ 1283 size = inputlen - iv_len; 1284 if (size) { 1285 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1286 i = fill_sg_comp_from_buf_min( 1287 gather_comp, i, 1288 fc_params->bufs, 1289 &size); 1290 } else { 1291 uint32_t aad_offset = aad_len ? 1292 passthrough_len : 0; 1293 1294 if (unlikely(!fc_params->src_iov)) { 1295 CPT_LOG_DP_ERR("Bad input args"); 1296 return; 1297 } 1298 1299 i = fill_sg_comp_from_iov( 1300 gather_comp, i, 1301 fc_params->src_iov, 1302 0, &size, 1303 aad_buf, 1304 aad_offset); 1305 } 1306 1307 if (unlikely(size)) { 1308 CPT_LOG_DP_ERR("Insufficient buffer" 1309 " space, size %d needed", 1310 size); 1311 return; 1312 } 1313 } 1314 } 1315 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1316 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1317 1318 /* 1319 * Output Scatter List 1320 */ 1321 1322 i = 0; 1323 scatter_comp = 1324 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1325 1326 /* Add iv */ 1327 if (iv_len) { 1328 i = fill_sg_comp(scatter_comp, i, 1329 offset_dma + OFF_CTRL_LEN, 1330 iv_len); 1331 } 1332 1333 /* Add output data */ 1334 size = outputlen - iv_len; 1335 if (size) { 1336 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1337 /* handle single buffer here */ 1338 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1339 fc_params->bufs, 1340 &size); 1341 } else { 1342 uint32_t aad_offset = aad_len ? 1343 passthrough_len : 0; 1344 1345 if (unlikely(!fc_params->dst_iov)) { 1346 CPT_LOG_DP_ERR("Bad input args"); 1347 return; 1348 } 1349 1350 i = fill_sg_comp_from_iov(scatter_comp, i, 1351 fc_params->dst_iov, 0, 1352 &size, aad_buf, 1353 aad_offset); 1354 } 1355 1356 if (unlikely(size)) { 1357 CPT_LOG_DP_ERR("Insufficient buffer space," 1358 " size %d needed", size); 1359 return; 1360 } 1361 } 1362 1363 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1364 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1365 1366 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1367 1368 /* This is DPTR len incase of SG mode */ 1369 vq_cmd_w0.s.dlen = size; 1370 1371 m_vaddr = (uint8_t *)m_vaddr + size; 1372 m_dma += size; 1373 1374 /* cpt alternate completion address saved earlier */ 1375 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1376 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1377 rptr_dma = c_dma - 8; 1378 size += COMPLETION_CODE_SIZE; 1379 1380 req->ist.ei1 = dptr_dma; 1381 req->ist.ei2 = rptr_dma; 1382 } 1383 1384 ctx_dma = fc_params->ctx_buf.dma_addr + 1385 offsetof(struct cpt_ctx, fctx); 1386 /* vq command w3 */ 1387 vq_cmd_w3.u64 = 0; 1388 vq_cmd_w3.s.grp = 0; 1389 vq_cmd_w3.s.cptr = ctx_dma; 1390 1391 /* 16 byte aligned cpt res address */ 1392 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1393 *req->completion_addr = COMPLETION_CODE_INIT; 1394 req->comp_baddr = c_dma; 1395 1396 /* Fill microcode part of instruction */ 1397 req->ist.ei0 = vq_cmd_w0.u64; 1398 req->ist.ei3 = vq_cmd_w3.u64; 1399 1400 req->op = op; 1401 1402 *prep_req = req; 1403 return; 1404 } 1405 1406 static __rte_always_inline void 1407 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1408 uint64_t d_offs, 1409 uint64_t d_lens, 1410 fc_params_t *params, 1411 void *op, 1412 void **prep_req) 1413 { 1414 uint32_t size; 1415 int32_t inputlen, outputlen; 1416 struct cpt_ctx *cpt_ctx; 1417 uint32_t mac_len = 0; 1418 uint8_t snow3g, j; 1419 struct cpt_request_info *req; 1420 buf_ptr_t *buf_p; 1421 uint32_t encr_offset = 0, auth_offset = 0; 1422 uint32_t encr_data_len = 0, auth_data_len = 0; 1423 int flags, iv_len = 16; 1424 void *m_vaddr, *c_vaddr; 1425 uint64_t m_dma, c_dma, offset_ctrl; 1426 uint64_t *offset_vaddr, offset_dma; 1427 uint32_t *iv_s, iv[4]; 1428 vq_cmd_word0_t vq_cmd_w0; 1429 vq_cmd_word3_t vq_cmd_w3; 1430 opcode_info_t opcode; 1431 1432 buf_p = ¶ms->meta_buf; 1433 m_vaddr = buf_p->vaddr; 1434 m_dma = buf_p->dma_addr; 1435 1436 cpt_ctx = params->ctx_buf.vaddr; 1437 flags = cpt_ctx->zsk_flags; 1438 mac_len = cpt_ctx->mac_len; 1439 snow3g = cpt_ctx->snow3g; 1440 1441 /* 1442 * Save initial space that followed app data for completion code & 1443 * alternate completion code to fall in same cache line as app data 1444 */ 1445 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1446 m_dma += COMPLETION_CODE_SIZE; 1447 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1448 (uint8_t *)m_vaddr; 1449 1450 c_vaddr = (uint8_t *)m_vaddr + size; 1451 c_dma = m_dma + size; 1452 size += sizeof(cpt_res_s_t); 1453 1454 m_vaddr = (uint8_t *)m_vaddr + size; 1455 m_dma += size; 1456 1457 /* Reserve memory for cpt request info */ 1458 req = m_vaddr; 1459 1460 size = sizeof(struct cpt_request_info); 1461 m_vaddr = (uint8_t *)m_vaddr + size; 1462 m_dma += size; 1463 1464 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1465 1466 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1467 1468 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1469 (0 << 3) | (flags & 0x7)); 1470 1471 if (flags == 0x1) { 1472 /* 1473 * Microcode expects offsets in bytes 1474 * TODO: Rounding off 1475 */ 1476 auth_data_len = AUTH_DLEN(d_lens); 1477 1478 /* EIA3 or UIA2 */ 1479 auth_offset = AUTH_OFFSET(d_offs); 1480 auth_offset = auth_offset / 8; 1481 1482 /* consider iv len */ 1483 auth_offset += iv_len; 1484 1485 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1486 outputlen = mac_len; 1487 1488 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1489 1490 } else { 1491 /* EEA3 or UEA2 */ 1492 /* 1493 * Microcode expects offsets in bytes 1494 * TODO: Rounding off 1495 */ 1496 encr_data_len = ENCR_DLEN(d_lens); 1497 1498 encr_offset = ENCR_OFFSET(d_offs); 1499 encr_offset = encr_offset / 8; 1500 /* consider iv len */ 1501 encr_offset += iv_len; 1502 1503 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1504 outputlen = inputlen; 1505 1506 /* iv offset is 0 */ 1507 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1508 } 1509 1510 /* IV */ 1511 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1512 params->iv_buf; 1513 1514 if (snow3g) { 1515 /* 1516 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1517 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1518 */ 1519 1520 for (j = 0; j < 4; j++) 1521 iv[j] = iv_s[3 - j]; 1522 } else { 1523 /* ZUC doesn't need a swap */ 1524 for (j = 0; j < 4; j++) 1525 iv[j] = iv_s[j]; 1526 } 1527 1528 /* 1529 * GP op header, lengths are expected in bits. 1530 */ 1531 vq_cmd_w0.u64 = 0; 1532 vq_cmd_w0.s.param1 = encr_data_len; 1533 vq_cmd_w0.s.param2 = auth_data_len; 1534 1535 /* 1536 * In 83XX since we have a limitation of 1537 * IV & Offset control word not part of instruction 1538 * and need to be part of Data Buffer, we check if 1539 * head room is there and then only do the Direct mode processing 1540 */ 1541 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1542 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1543 void *dm_vaddr = params->bufs[0].vaddr; 1544 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1545 /* 1546 * This flag indicates that there is 24 bytes head room and 1547 * 8 bytes tail room available, so that we get to do 1548 * DIRECT MODE with limitation 1549 */ 1550 1551 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1552 OFF_CTRL_LEN - iv_len); 1553 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1554 1555 /* DPTR */ 1556 req->ist.ei1 = offset_dma; 1557 /* RPTR should just exclude offset control word */ 1558 req->ist.ei2 = dm_dma_addr - iv_len; 1559 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1560 + outputlen - iv_len); 1561 1562 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1563 1564 vq_cmd_w0.s.opcode = opcode.flags; 1565 1566 if (likely(iv_len)) { 1567 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1568 + OFF_CTRL_LEN); 1569 memcpy(iv_d, iv, 16); 1570 } 1571 1572 *offset_vaddr = offset_ctrl; 1573 } else { 1574 uint32_t i, g_size_bytes, s_size_bytes; 1575 uint64_t dptr_dma, rptr_dma; 1576 sg_comp_t *gather_comp; 1577 sg_comp_t *scatter_comp; 1578 uint8_t *in_buffer; 1579 uint32_t *iv_d; 1580 1581 /* save space for iv */ 1582 offset_vaddr = m_vaddr; 1583 offset_dma = m_dma; 1584 1585 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1586 m_dma += OFF_CTRL_LEN + iv_len; 1587 1588 opcode.s.major |= CPT_DMA_MODE; 1589 1590 vq_cmd_w0.s.opcode = opcode.flags; 1591 1592 /* DPTR has SG list */ 1593 in_buffer = m_vaddr; 1594 dptr_dma = m_dma; 1595 1596 ((uint16_t *)in_buffer)[0] = 0; 1597 ((uint16_t *)in_buffer)[1] = 0; 1598 1599 /* TODO Add error check if space will be sufficient */ 1600 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1601 1602 /* 1603 * Input Gather List 1604 */ 1605 i = 0; 1606 1607 /* Offset control word followed by iv */ 1608 1609 i = fill_sg_comp(gather_comp, i, offset_dma, 1610 OFF_CTRL_LEN + iv_len); 1611 1612 /* iv offset is 0 */ 1613 *offset_vaddr = offset_ctrl; 1614 1615 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1616 memcpy(iv_d, iv, 16); 1617 1618 /* input data */ 1619 size = inputlen - iv_len; 1620 if (size) { 1621 i = fill_sg_comp_from_iov(gather_comp, i, 1622 params->src_iov, 1623 0, &size, NULL, 0); 1624 if (unlikely(size)) { 1625 CPT_LOG_DP_ERR("Insufficient buffer space," 1626 " size %d needed", size); 1627 return; 1628 } 1629 } 1630 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1631 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1632 1633 /* 1634 * Output Scatter List 1635 */ 1636 1637 i = 0; 1638 scatter_comp = 1639 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1640 1641 if (flags == 0x1) { 1642 /* IV in SLIST only for EEA3 & UEA2 */ 1643 iv_len = 0; 1644 } 1645 1646 if (iv_len) { 1647 i = fill_sg_comp(scatter_comp, i, 1648 offset_dma + OFF_CTRL_LEN, iv_len); 1649 } 1650 1651 /* Add output data */ 1652 if (req_flags & VALID_MAC_BUF) { 1653 size = outputlen - iv_len - mac_len; 1654 if (size) { 1655 i = fill_sg_comp_from_iov(scatter_comp, i, 1656 params->dst_iov, 0, 1657 &size, NULL, 0); 1658 1659 if (unlikely(size)) { 1660 CPT_LOG_DP_ERR("Insufficient buffer space," 1661 " size %d needed", size); 1662 return; 1663 } 1664 } 1665 1666 /* mac data */ 1667 if (mac_len) { 1668 i = fill_sg_comp_from_buf(scatter_comp, i, 1669 ¶ms->mac_buf); 1670 } 1671 } else { 1672 /* Output including mac */ 1673 size = outputlen - iv_len; 1674 if (size) { 1675 i = fill_sg_comp_from_iov(scatter_comp, i, 1676 params->dst_iov, 0, 1677 &size, NULL, 0); 1678 1679 if (unlikely(size)) { 1680 CPT_LOG_DP_ERR("Insufficient buffer space," 1681 " size %d needed", size); 1682 return; 1683 } 1684 } 1685 } 1686 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1687 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1688 1689 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1690 1691 /* This is DPTR len incase of SG mode */ 1692 vq_cmd_w0.s.dlen = size; 1693 1694 m_vaddr = (uint8_t *)m_vaddr + size; 1695 m_dma += size; 1696 1697 /* cpt alternate completion address saved earlier */ 1698 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1699 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1700 rptr_dma = c_dma - 8; 1701 1702 req->ist.ei1 = dptr_dma; 1703 req->ist.ei2 = rptr_dma; 1704 } 1705 1706 /* vq command w3 */ 1707 vq_cmd_w3.u64 = 0; 1708 vq_cmd_w3.s.grp = 0; 1709 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1710 offsetof(struct cpt_ctx, zs_ctx); 1711 1712 /* 16 byte aligned cpt res address */ 1713 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1714 *req->completion_addr = COMPLETION_CODE_INIT; 1715 req->comp_baddr = c_dma; 1716 1717 /* Fill microcode part of instruction */ 1718 req->ist.ei0 = vq_cmd_w0.u64; 1719 req->ist.ei3 = vq_cmd_w3.u64; 1720 1721 req->op = op; 1722 1723 *prep_req = req; 1724 return; 1725 } 1726 1727 static __rte_always_inline void 1728 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1729 uint64_t d_offs, 1730 uint64_t d_lens, 1731 fc_params_t *params, 1732 void *op, 1733 void **prep_req) 1734 { 1735 uint32_t size; 1736 int32_t inputlen = 0, outputlen; 1737 struct cpt_ctx *cpt_ctx; 1738 uint8_t snow3g, iv_len = 16; 1739 struct cpt_request_info *req; 1740 buf_ptr_t *buf_p; 1741 uint32_t encr_offset; 1742 uint32_t encr_data_len; 1743 int flags; 1744 void *m_vaddr, *c_vaddr; 1745 uint64_t m_dma, c_dma; 1746 uint64_t *offset_vaddr, offset_dma; 1747 uint32_t *iv_s, iv[4], j; 1748 vq_cmd_word0_t vq_cmd_w0; 1749 vq_cmd_word3_t vq_cmd_w3; 1750 opcode_info_t opcode; 1751 1752 buf_p = ¶ms->meta_buf; 1753 m_vaddr = buf_p->vaddr; 1754 m_dma = buf_p->dma_addr; 1755 1756 /* 1757 * Microcode expects offsets in bytes 1758 * TODO: Rounding off 1759 */ 1760 encr_offset = ENCR_OFFSET(d_offs) / 8; 1761 encr_data_len = ENCR_DLEN(d_lens); 1762 1763 cpt_ctx = params->ctx_buf.vaddr; 1764 flags = cpt_ctx->zsk_flags; 1765 snow3g = cpt_ctx->snow3g; 1766 /* 1767 * Save initial space that followed app data for completion code & 1768 * alternate completion code to fall in same cache line as app data 1769 */ 1770 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1771 m_dma += COMPLETION_CODE_SIZE; 1772 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1773 (uint8_t *)m_vaddr; 1774 1775 c_vaddr = (uint8_t *)m_vaddr + size; 1776 c_dma = m_dma + size; 1777 size += sizeof(cpt_res_s_t); 1778 1779 m_vaddr = (uint8_t *)m_vaddr + size; 1780 m_dma += size; 1781 1782 /* Reserve memory for cpt request info */ 1783 req = m_vaddr; 1784 1785 size = sizeof(struct cpt_request_info); 1786 m_vaddr = (uint8_t *)m_vaddr + size; 1787 m_dma += size; 1788 1789 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1790 1791 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1792 1793 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1794 (0 << 3) | (flags & 0x7)); 1795 1796 /* consider iv len */ 1797 encr_offset += iv_len; 1798 1799 inputlen = encr_offset + 1800 (RTE_ALIGN(encr_data_len, 8) / 8); 1801 outputlen = inputlen; 1802 1803 /* IV */ 1804 iv_s = params->iv_buf; 1805 if (snow3g) { 1806 /* 1807 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1808 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1809 */ 1810 1811 for (j = 0; j < 4; j++) 1812 iv[j] = iv_s[3 - j]; 1813 } else { 1814 /* ZUC doesn't need a swap */ 1815 for (j = 0; j < 4; j++) 1816 iv[j] = iv_s[j]; 1817 } 1818 1819 /* 1820 * GP op header, lengths are expected in bits. 1821 */ 1822 vq_cmd_w0.u64 = 0; 1823 vq_cmd_w0.s.param1 = encr_data_len; 1824 1825 /* 1826 * In 83XX since we have a limitation of 1827 * IV & Offset control word not part of instruction 1828 * and need to be part of Data Buffer, we check if 1829 * head room is there and then only do the Direct mode processing 1830 */ 1831 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1832 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1833 void *dm_vaddr = params->bufs[0].vaddr; 1834 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1835 /* 1836 * This flag indicates that there is 24 bytes head room and 1837 * 8 bytes tail room available, so that we get to do 1838 * DIRECT MODE with limitation 1839 */ 1840 1841 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1842 OFF_CTRL_LEN - iv_len); 1843 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1844 1845 /* DPTR */ 1846 req->ist.ei1 = offset_dma; 1847 /* RPTR should just exclude offset control word */ 1848 req->ist.ei2 = dm_dma_addr - iv_len; 1849 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1850 + outputlen - iv_len); 1851 1852 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1853 1854 vq_cmd_w0.s.opcode = opcode.flags; 1855 1856 if (likely(iv_len)) { 1857 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1858 + OFF_CTRL_LEN); 1859 memcpy(iv_d, iv, 16); 1860 } 1861 1862 /* iv offset is 0 */ 1863 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1864 } else { 1865 uint32_t i, g_size_bytes, s_size_bytes; 1866 uint64_t dptr_dma, rptr_dma; 1867 sg_comp_t *gather_comp; 1868 sg_comp_t *scatter_comp; 1869 uint8_t *in_buffer; 1870 uint32_t *iv_d; 1871 1872 /* save space for offset and iv... */ 1873 offset_vaddr = m_vaddr; 1874 offset_dma = m_dma; 1875 1876 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1877 m_dma += OFF_CTRL_LEN + iv_len; 1878 1879 opcode.s.major |= CPT_DMA_MODE; 1880 1881 vq_cmd_w0.s.opcode = opcode.flags; 1882 1883 /* DPTR has SG list */ 1884 in_buffer = m_vaddr; 1885 dptr_dma = m_dma; 1886 1887 ((uint16_t *)in_buffer)[0] = 0; 1888 ((uint16_t *)in_buffer)[1] = 0; 1889 1890 /* TODO Add error check if space will be sufficient */ 1891 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1892 1893 /* 1894 * Input Gather List 1895 */ 1896 i = 0; 1897 1898 /* Offset control word */ 1899 1900 /* iv offset is 0 */ 1901 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1902 1903 i = fill_sg_comp(gather_comp, i, offset_dma, 1904 OFF_CTRL_LEN + iv_len); 1905 1906 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1907 memcpy(iv_d, iv, 16); 1908 1909 /* Add input data */ 1910 size = inputlen - iv_len; 1911 if (size) { 1912 i = fill_sg_comp_from_iov(gather_comp, i, 1913 params->src_iov, 1914 0, &size, NULL, 0); 1915 if (unlikely(size)) { 1916 CPT_LOG_DP_ERR("Insufficient buffer space," 1917 " size %d needed", size); 1918 return; 1919 } 1920 } 1921 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1922 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1923 1924 /* 1925 * Output Scatter List 1926 */ 1927 1928 i = 0; 1929 scatter_comp = 1930 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1931 1932 /* IV */ 1933 i = fill_sg_comp(scatter_comp, i, 1934 offset_dma + OFF_CTRL_LEN, 1935 iv_len); 1936 1937 /* Add output data */ 1938 size = outputlen - iv_len; 1939 if (size) { 1940 i = fill_sg_comp_from_iov(scatter_comp, i, 1941 params->dst_iov, 0, 1942 &size, NULL, 0); 1943 1944 if (unlikely(size)) { 1945 CPT_LOG_DP_ERR("Insufficient buffer space," 1946 " size %d needed", size); 1947 return; 1948 } 1949 } 1950 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1951 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1952 1953 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1954 1955 /* This is DPTR len incase of SG mode */ 1956 vq_cmd_w0.s.dlen = size; 1957 1958 m_vaddr = (uint8_t *)m_vaddr + size; 1959 m_dma += size; 1960 1961 /* cpt alternate completion address saved earlier */ 1962 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1963 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1964 rptr_dma = c_dma - 8; 1965 1966 req->ist.ei1 = dptr_dma; 1967 req->ist.ei2 = rptr_dma; 1968 } 1969 1970 /* vq command w3 */ 1971 vq_cmd_w3.u64 = 0; 1972 vq_cmd_w3.s.grp = 0; 1973 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1974 offsetof(struct cpt_ctx, zs_ctx); 1975 1976 /* 16 byte aligned cpt res address */ 1977 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1978 *req->completion_addr = COMPLETION_CODE_INIT; 1979 req->comp_baddr = c_dma; 1980 1981 /* Fill microcode part of instruction */ 1982 req->ist.ei0 = vq_cmd_w0.u64; 1983 req->ist.ei3 = vq_cmd_w3.u64; 1984 1985 req->op = op; 1986 1987 *prep_req = req; 1988 return; 1989 } 1990 1991 static __rte_always_inline void 1992 cpt_kasumi_enc_prep(uint32_t req_flags, 1993 uint64_t d_offs, 1994 uint64_t d_lens, 1995 fc_params_t *params, 1996 void *op, 1997 void **prep_req) 1998 { 1999 uint32_t size; 2000 int32_t inputlen = 0, outputlen = 0; 2001 struct cpt_ctx *cpt_ctx; 2002 uint32_t mac_len = 0; 2003 uint8_t i = 0; 2004 struct cpt_request_info *req; 2005 buf_ptr_t *buf_p; 2006 uint32_t encr_offset, auth_offset; 2007 uint32_t encr_data_len, auth_data_len; 2008 int flags; 2009 uint8_t *iv_s, *iv_d, iv_len = 8; 2010 uint8_t dir = 0; 2011 void *m_vaddr, *c_vaddr; 2012 uint64_t m_dma, c_dma; 2013 uint64_t *offset_vaddr, offset_dma; 2014 vq_cmd_word0_t vq_cmd_w0; 2015 vq_cmd_word3_t vq_cmd_w3; 2016 opcode_info_t opcode; 2017 uint8_t *in_buffer; 2018 uint32_t g_size_bytes, s_size_bytes; 2019 uint64_t dptr_dma, rptr_dma; 2020 sg_comp_t *gather_comp; 2021 sg_comp_t *scatter_comp; 2022 2023 buf_p = ¶ms->meta_buf; 2024 m_vaddr = buf_p->vaddr; 2025 m_dma = buf_p->dma_addr; 2026 2027 encr_offset = ENCR_OFFSET(d_offs) / 8; 2028 auth_offset = AUTH_OFFSET(d_offs) / 8; 2029 encr_data_len = ENCR_DLEN(d_lens); 2030 auth_data_len = AUTH_DLEN(d_lens); 2031 2032 cpt_ctx = params->ctx_buf.vaddr; 2033 flags = cpt_ctx->zsk_flags; 2034 mac_len = cpt_ctx->mac_len; 2035 2036 if (flags == 0x0) 2037 iv_s = params->iv_buf; 2038 else 2039 iv_s = params->auth_iv_buf; 2040 2041 dir = iv_s[8] & 0x1; 2042 2043 /* 2044 * Save initial space that followed app data for completion code & 2045 * alternate completion code to fall in same cache line as app data 2046 */ 2047 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2048 m_dma += COMPLETION_CODE_SIZE; 2049 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2050 (uint8_t *)m_vaddr; 2051 2052 c_vaddr = (uint8_t *)m_vaddr + size; 2053 c_dma = m_dma + size; 2054 size += sizeof(cpt_res_s_t); 2055 2056 m_vaddr = (uint8_t *)m_vaddr + size; 2057 m_dma += size; 2058 2059 /* Reserve memory for cpt request info */ 2060 req = m_vaddr; 2061 2062 size = sizeof(struct cpt_request_info); 2063 m_vaddr = (uint8_t *)m_vaddr + size; 2064 m_dma += size; 2065 2066 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2067 2068 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2069 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2070 (dir << 4) | (0 << 3) | (flags & 0x7)); 2071 2072 /* 2073 * GP op header, lengths are expected in bits. 2074 */ 2075 vq_cmd_w0.u64 = 0; 2076 vq_cmd_w0.s.param1 = encr_data_len; 2077 vq_cmd_w0.s.param2 = auth_data_len; 2078 vq_cmd_w0.s.opcode = opcode.flags; 2079 2080 /* consider iv len */ 2081 if (flags == 0x0) { 2082 encr_offset += iv_len; 2083 auth_offset += iv_len; 2084 } 2085 2086 /* save space for offset ctrl and iv */ 2087 offset_vaddr = m_vaddr; 2088 offset_dma = m_dma; 2089 2090 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2091 m_dma += OFF_CTRL_LEN + iv_len; 2092 2093 /* DPTR has SG list */ 2094 in_buffer = m_vaddr; 2095 dptr_dma = m_dma; 2096 2097 ((uint16_t *)in_buffer)[0] = 0; 2098 ((uint16_t *)in_buffer)[1] = 0; 2099 2100 /* TODO Add error check if space will be sufficient */ 2101 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2102 2103 /* 2104 * Input Gather List 2105 */ 2106 i = 0; 2107 2108 /* Offset control word followed by iv */ 2109 2110 if (flags == 0x0) { 2111 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2112 outputlen = inputlen; 2113 /* iv offset is 0 */ 2114 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2115 } else { 2116 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2117 outputlen = mac_len; 2118 /* iv offset is 0 */ 2119 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2120 } 2121 2122 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2123 2124 /* IV */ 2125 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2126 memcpy(iv_d, iv_s, iv_len); 2127 2128 /* input data */ 2129 size = inputlen - iv_len; 2130 if (size) { 2131 i = fill_sg_comp_from_iov(gather_comp, i, 2132 params->src_iov, 0, 2133 &size, NULL, 0); 2134 2135 if (unlikely(size)) { 2136 CPT_LOG_DP_ERR("Insufficient buffer space," 2137 " size %d needed", size); 2138 return; 2139 } 2140 } 2141 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2142 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2143 2144 /* 2145 * Output Scatter List 2146 */ 2147 2148 i = 0; 2149 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2150 2151 if (flags == 0x1) { 2152 /* IV in SLIST only for F8 */ 2153 iv_len = 0; 2154 } 2155 2156 /* IV */ 2157 if (iv_len) { 2158 i = fill_sg_comp(scatter_comp, i, 2159 offset_dma + OFF_CTRL_LEN, 2160 iv_len); 2161 } 2162 2163 /* Add output data */ 2164 if (req_flags & VALID_MAC_BUF) { 2165 size = outputlen - iv_len - mac_len; 2166 if (size) { 2167 i = fill_sg_comp_from_iov(scatter_comp, i, 2168 params->dst_iov, 0, 2169 &size, NULL, 0); 2170 2171 if (unlikely(size)) { 2172 CPT_LOG_DP_ERR("Insufficient buffer space," 2173 " size %d needed", size); 2174 return; 2175 } 2176 } 2177 2178 /* mac data */ 2179 if (mac_len) { 2180 i = fill_sg_comp_from_buf(scatter_comp, i, 2181 ¶ms->mac_buf); 2182 } 2183 } else { 2184 /* Output including mac */ 2185 size = outputlen - iv_len; 2186 if (size) { 2187 i = fill_sg_comp_from_iov(scatter_comp, i, 2188 params->dst_iov, 0, 2189 &size, NULL, 0); 2190 2191 if (unlikely(size)) { 2192 CPT_LOG_DP_ERR("Insufficient buffer space," 2193 " size %d needed", size); 2194 return; 2195 } 2196 } 2197 } 2198 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2199 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2200 2201 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2202 2203 /* This is DPTR len incase of SG mode */ 2204 vq_cmd_w0.s.dlen = size; 2205 2206 m_vaddr = (uint8_t *)m_vaddr + size; 2207 m_dma += size; 2208 2209 /* cpt alternate completion address saved earlier */ 2210 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2211 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2212 rptr_dma = c_dma - 8; 2213 2214 req->ist.ei1 = dptr_dma; 2215 req->ist.ei2 = rptr_dma; 2216 2217 /* vq command w3 */ 2218 vq_cmd_w3.u64 = 0; 2219 vq_cmd_w3.s.grp = 0; 2220 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2221 offsetof(struct cpt_ctx, k_ctx); 2222 2223 /* 16 byte aligned cpt res address */ 2224 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2225 *req->completion_addr = COMPLETION_CODE_INIT; 2226 req->comp_baddr = c_dma; 2227 2228 /* Fill microcode part of instruction */ 2229 req->ist.ei0 = vq_cmd_w0.u64; 2230 req->ist.ei3 = vq_cmd_w3.u64; 2231 2232 req->op = op; 2233 2234 *prep_req = req; 2235 return; 2236 } 2237 2238 static __rte_always_inline void 2239 cpt_kasumi_dec_prep(uint64_t d_offs, 2240 uint64_t d_lens, 2241 fc_params_t *params, 2242 void *op, 2243 void **prep_req) 2244 { 2245 uint32_t size; 2246 int32_t inputlen = 0, outputlen; 2247 struct cpt_ctx *cpt_ctx; 2248 uint8_t i = 0, iv_len = 8; 2249 struct cpt_request_info *req; 2250 buf_ptr_t *buf_p; 2251 uint32_t encr_offset; 2252 uint32_t encr_data_len; 2253 int flags; 2254 uint8_t dir = 0; 2255 void *m_vaddr, *c_vaddr; 2256 uint64_t m_dma, c_dma; 2257 uint64_t *offset_vaddr, offset_dma; 2258 vq_cmd_word0_t vq_cmd_w0; 2259 vq_cmd_word3_t vq_cmd_w3; 2260 opcode_info_t opcode; 2261 uint8_t *in_buffer; 2262 uint32_t g_size_bytes, s_size_bytes; 2263 uint64_t dptr_dma, rptr_dma; 2264 sg_comp_t *gather_comp; 2265 sg_comp_t *scatter_comp; 2266 2267 buf_p = ¶ms->meta_buf; 2268 m_vaddr = buf_p->vaddr; 2269 m_dma = buf_p->dma_addr; 2270 2271 encr_offset = ENCR_OFFSET(d_offs) / 8; 2272 encr_data_len = ENCR_DLEN(d_lens); 2273 2274 cpt_ctx = params->ctx_buf.vaddr; 2275 flags = cpt_ctx->zsk_flags; 2276 /* 2277 * Save initial space that followed app data for completion code & 2278 * alternate completion code to fall in same cache line as app data 2279 */ 2280 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2281 m_dma += COMPLETION_CODE_SIZE; 2282 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2283 (uint8_t *)m_vaddr; 2284 2285 c_vaddr = (uint8_t *)m_vaddr + size; 2286 c_dma = m_dma + size; 2287 size += sizeof(cpt_res_s_t); 2288 2289 m_vaddr = (uint8_t *)m_vaddr + size; 2290 m_dma += size; 2291 2292 /* Reserve memory for cpt request info */ 2293 req = m_vaddr; 2294 2295 size = sizeof(struct cpt_request_info); 2296 m_vaddr = (uint8_t *)m_vaddr + size; 2297 m_dma += size; 2298 2299 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2300 2301 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2302 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2303 (dir << 4) | (0 << 3) | (flags & 0x7)); 2304 2305 /* 2306 * GP op header, lengths are expected in bits. 2307 */ 2308 vq_cmd_w0.u64 = 0; 2309 vq_cmd_w0.s.param1 = encr_data_len; 2310 vq_cmd_w0.s.opcode = opcode.flags; 2311 2312 /* consider iv len */ 2313 encr_offset += iv_len; 2314 2315 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2316 outputlen = inputlen; 2317 2318 /* save space for offset ctrl & iv */ 2319 offset_vaddr = m_vaddr; 2320 offset_dma = m_dma; 2321 2322 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2323 m_dma += OFF_CTRL_LEN + iv_len; 2324 2325 /* DPTR has SG list */ 2326 in_buffer = m_vaddr; 2327 dptr_dma = m_dma; 2328 2329 ((uint16_t *)in_buffer)[0] = 0; 2330 ((uint16_t *)in_buffer)[1] = 0; 2331 2332 /* TODO Add error check if space will be sufficient */ 2333 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2334 2335 /* 2336 * Input Gather List 2337 */ 2338 i = 0; 2339 2340 /* Offset control word followed by iv */ 2341 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2342 2343 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2344 2345 /* IV */ 2346 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2347 params->iv_buf, iv_len); 2348 2349 /* Add input data */ 2350 size = inputlen - iv_len; 2351 if (size) { 2352 i = fill_sg_comp_from_iov(gather_comp, i, 2353 params->src_iov, 2354 0, &size, NULL, 0); 2355 if (unlikely(size)) { 2356 CPT_LOG_DP_ERR("Insufficient buffer space," 2357 " size %d needed", size); 2358 return; 2359 } 2360 } 2361 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2362 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2363 2364 /* 2365 * Output Scatter List 2366 */ 2367 2368 i = 0; 2369 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2370 2371 /* IV */ 2372 i = fill_sg_comp(scatter_comp, i, 2373 offset_dma + OFF_CTRL_LEN, 2374 iv_len); 2375 2376 /* Add output data */ 2377 size = outputlen - iv_len; 2378 if (size) { 2379 i = fill_sg_comp_from_iov(scatter_comp, i, 2380 params->dst_iov, 0, 2381 &size, NULL, 0); 2382 if (unlikely(size)) { 2383 CPT_LOG_DP_ERR("Insufficient buffer space," 2384 " size %d needed", size); 2385 return; 2386 } 2387 } 2388 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2389 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2390 2391 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2392 2393 /* This is DPTR len incase of SG mode */ 2394 vq_cmd_w0.s.dlen = size; 2395 2396 m_vaddr = (uint8_t *)m_vaddr + size; 2397 m_dma += size; 2398 2399 /* cpt alternate completion address saved earlier */ 2400 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2401 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2402 rptr_dma = c_dma - 8; 2403 2404 req->ist.ei1 = dptr_dma; 2405 req->ist.ei2 = rptr_dma; 2406 2407 /* vq command w3 */ 2408 vq_cmd_w3.u64 = 0; 2409 vq_cmd_w3.s.grp = 0; 2410 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2411 offsetof(struct cpt_ctx, k_ctx); 2412 2413 /* 16 byte aligned cpt res address */ 2414 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2415 *req->completion_addr = COMPLETION_CODE_INIT; 2416 req->comp_baddr = c_dma; 2417 2418 /* Fill microcode part of instruction */ 2419 req->ist.ei0 = vq_cmd_w0.u64; 2420 req->ist.ei3 = vq_cmd_w3.u64; 2421 2422 req->op = op; 2423 2424 *prep_req = req; 2425 return; 2426 } 2427 2428 static __rte_always_inline void * 2429 cpt_fc_dec_hmac_prep(uint32_t flags, 2430 uint64_t d_offs, 2431 uint64_t d_lens, 2432 fc_params_t *fc_params, 2433 void *op) 2434 { 2435 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2436 uint8_t fc_type; 2437 void *prep_req = NULL; 2438 2439 fc_type = ctx->fc_type; 2440 2441 if (likely(fc_type == FC_GEN)) { 2442 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2443 &prep_req); 2444 } else if (fc_type == ZUC_SNOW3G) { 2445 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2446 &prep_req); 2447 } else if (fc_type == KASUMI) { 2448 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2449 } 2450 2451 /* 2452 * For AUTH_ONLY case, 2453 * MC only supports digest generation and verification 2454 * should be done in software by memcmp() 2455 */ 2456 2457 return prep_req; 2458 } 2459 2460 static __rte_always_inline void *__hot 2461 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2462 fc_params_t *fc_params, void *op) 2463 { 2464 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2465 uint8_t fc_type; 2466 void *prep_req = NULL; 2467 2468 fc_type = ctx->fc_type; 2469 2470 /* Common api for rest of the ops */ 2471 if (likely(fc_type == FC_GEN)) { 2472 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2473 &prep_req); 2474 } else if (fc_type == ZUC_SNOW3G) { 2475 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2476 &prep_req); 2477 } else if (fc_type == KASUMI) { 2478 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2479 &prep_req); 2480 } else if (fc_type == HASH_HMAC) { 2481 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2482 } 2483 2484 return prep_req; 2485 } 2486 2487 static __rte_always_inline int 2488 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2489 uint16_t key_len, uint16_t mac_len) 2490 { 2491 struct cpt_ctx *cpt_ctx = ctx; 2492 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2493 2494 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2495 uint32_t keyx[4]; 2496 2497 if (key_len != 16) 2498 return -1; 2499 /* No support for AEAD yet */ 2500 if (cpt_ctx->enc_cipher) 2501 return -1; 2502 /* For ZUC/SNOW3G/Kasumi */ 2503 switch (type) { 2504 case SNOW3G_UIA2: 2505 cpt_ctx->snow3g = 1; 2506 gen_key_snow3g(key, keyx); 2507 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2508 cpt_ctx->fc_type = ZUC_SNOW3G; 2509 cpt_ctx->zsk_flags = 0x1; 2510 break; 2511 case ZUC_EIA3: 2512 cpt_ctx->snow3g = 0; 2513 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2514 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2515 cpt_ctx->fc_type = ZUC_SNOW3G; 2516 cpt_ctx->zsk_flags = 0x1; 2517 break; 2518 case KASUMI_F9_ECB: 2519 /* Kasumi ECB mode */ 2520 cpt_ctx->k_ecb = 1; 2521 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2522 cpt_ctx->fc_type = KASUMI; 2523 cpt_ctx->zsk_flags = 0x1; 2524 break; 2525 case KASUMI_F9_CBC: 2526 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2527 cpt_ctx->fc_type = KASUMI; 2528 cpt_ctx->zsk_flags = 0x1; 2529 break; 2530 default: 2531 return -1; 2532 } 2533 cpt_ctx->mac_len = 4; 2534 cpt_ctx->hash_type = type; 2535 return 0; 2536 } 2537 2538 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2539 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2540 cpt_ctx->fc_type = HASH_HMAC; 2541 } 2542 2543 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2544 return -1; 2545 2546 /* For GMAC auth, cipher must be NULL */ 2547 if (type == GMAC_TYPE) 2548 fctx->enc.enc_cipher = 0; 2549 2550 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2551 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2552 2553 if (key_len) { 2554 cpt_ctx->hmac = 1; 2555 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2556 memcpy(cpt_ctx->auth_key, key, key_len); 2557 cpt_ctx->auth_key_len = key_len; 2558 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2559 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2560 2561 if (key_len <= 64) 2562 memcpy(fctx->hmac.opad, key, key_len); 2563 fctx->enc.auth_input_type = 1; 2564 } 2565 return 0; 2566 } 2567 2568 static __rte_always_inline int 2569 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2570 struct cpt_sess_misc *sess) 2571 { 2572 struct rte_crypto_aead_xform *aead_form; 2573 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2574 auth_type_t auth_type = 0; /* NULL Auth type */ 2575 uint32_t cipher_key_len = 0; 2576 uint8_t aes_gcm = 0; 2577 aead_form = &xform->aead; 2578 void *ctx = SESS_PRIV(sess); 2579 2580 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2581 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2582 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2583 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2584 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2585 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2586 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2587 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2588 } else { 2589 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2590 return -1; 2591 } 2592 switch (aead_form->algo) { 2593 case RTE_CRYPTO_AEAD_AES_GCM: 2594 enc_type = AES_GCM; 2595 cipher_key_len = 16; 2596 aes_gcm = 1; 2597 break; 2598 case RTE_CRYPTO_AEAD_AES_CCM: 2599 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2600 aead_form->algo); 2601 return -1; 2602 default: 2603 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2604 aead_form->algo); 2605 return -1; 2606 } 2607 if (aead_form->key.length < cipher_key_len) { 2608 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2609 (unsigned int long)aead_form->key.length); 2610 return -1; 2611 } 2612 sess->zsk_flag = 0; 2613 sess->aes_gcm = aes_gcm; 2614 sess->mac_len = aead_form->digest_length; 2615 sess->iv_offset = aead_form->iv.offset; 2616 sess->iv_length = aead_form->iv.length; 2617 sess->aad_length = aead_form->aad_length; 2618 2619 cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2620 aead_form->key.length, NULL); 2621 2622 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length); 2623 2624 return 0; 2625 } 2626 2627 static __rte_always_inline int 2628 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2629 struct cpt_sess_misc *sess) 2630 { 2631 struct rte_crypto_cipher_xform *c_form; 2632 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2633 uint32_t cipher_key_len = 0; 2634 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2635 2636 c_form = &xform->cipher; 2637 2638 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2639 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2640 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2641 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2642 else { 2643 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2644 return -1; 2645 } 2646 2647 switch (c_form->algo) { 2648 case RTE_CRYPTO_CIPHER_AES_CBC: 2649 enc_type = AES_CBC; 2650 cipher_key_len = 16; 2651 break; 2652 case RTE_CRYPTO_CIPHER_3DES_CBC: 2653 enc_type = DES3_CBC; 2654 cipher_key_len = 24; 2655 break; 2656 case RTE_CRYPTO_CIPHER_DES_CBC: 2657 /* DES is implemented using 3DES in hardware */ 2658 enc_type = DES3_CBC; 2659 cipher_key_len = 8; 2660 break; 2661 case RTE_CRYPTO_CIPHER_AES_CTR: 2662 enc_type = AES_CTR; 2663 cipher_key_len = 16; 2664 aes_ctr = 1; 2665 break; 2666 case RTE_CRYPTO_CIPHER_NULL: 2667 enc_type = 0; 2668 is_null = 1; 2669 break; 2670 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2671 enc_type = KASUMI_F8_ECB; 2672 cipher_key_len = 16; 2673 zsk_flag = K_F8; 2674 break; 2675 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2676 enc_type = SNOW3G_UEA2; 2677 cipher_key_len = 16; 2678 zsk_flag = ZS_EA; 2679 break; 2680 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2681 enc_type = ZUC_EEA3; 2682 cipher_key_len = 16; 2683 zsk_flag = ZS_EA; 2684 break; 2685 case RTE_CRYPTO_CIPHER_AES_XTS: 2686 enc_type = AES_XTS; 2687 cipher_key_len = 16; 2688 break; 2689 case RTE_CRYPTO_CIPHER_3DES_ECB: 2690 enc_type = DES3_ECB; 2691 cipher_key_len = 24; 2692 break; 2693 case RTE_CRYPTO_CIPHER_AES_ECB: 2694 enc_type = AES_ECB; 2695 cipher_key_len = 16; 2696 break; 2697 case RTE_CRYPTO_CIPHER_3DES_CTR: 2698 case RTE_CRYPTO_CIPHER_AES_F8: 2699 case RTE_CRYPTO_CIPHER_ARC4: 2700 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2701 c_form->algo); 2702 return -1; 2703 default: 2704 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2705 c_form->algo); 2706 return -1; 2707 } 2708 2709 if (c_form->key.length < cipher_key_len) { 2710 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2711 (unsigned long) c_form->key.length); 2712 return -1; 2713 } 2714 2715 sess->zsk_flag = zsk_flag; 2716 sess->aes_gcm = 0; 2717 sess->aes_ctr = aes_ctr; 2718 sess->iv_offset = c_form->iv.offset; 2719 sess->iv_length = c_form->iv.length; 2720 sess->is_null = is_null; 2721 2722 cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data, 2723 c_form->key.length, NULL); 2724 2725 return 0; 2726 } 2727 2728 static __rte_always_inline int 2729 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2730 struct cpt_sess_misc *sess) 2731 { 2732 struct rte_crypto_auth_xform *a_form; 2733 auth_type_t auth_type = 0; /* NULL Auth type */ 2734 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2735 2736 a_form = &xform->auth; 2737 2738 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2739 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2740 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2741 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2742 else { 2743 CPT_LOG_DP_ERR("Unknown auth operation"); 2744 return -1; 2745 } 2746 2747 switch (a_form->algo) { 2748 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2749 /* Fall through */ 2750 case RTE_CRYPTO_AUTH_SHA1: 2751 auth_type = SHA1_TYPE; 2752 break; 2753 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2754 case RTE_CRYPTO_AUTH_SHA256: 2755 auth_type = SHA2_SHA256; 2756 break; 2757 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2758 case RTE_CRYPTO_AUTH_SHA512: 2759 auth_type = SHA2_SHA512; 2760 break; 2761 case RTE_CRYPTO_AUTH_AES_GMAC: 2762 auth_type = GMAC_TYPE; 2763 aes_gcm = 1; 2764 break; 2765 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2766 case RTE_CRYPTO_AUTH_SHA224: 2767 auth_type = SHA2_SHA224; 2768 break; 2769 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2770 case RTE_CRYPTO_AUTH_SHA384: 2771 auth_type = SHA2_SHA384; 2772 break; 2773 case RTE_CRYPTO_AUTH_MD5_HMAC: 2774 case RTE_CRYPTO_AUTH_MD5: 2775 auth_type = MD5_TYPE; 2776 break; 2777 case RTE_CRYPTO_AUTH_KASUMI_F9: 2778 auth_type = KASUMI_F9_ECB; 2779 /* 2780 * Indicate that direction needs to be taken out 2781 * from end of src 2782 */ 2783 zsk_flag = K_F9; 2784 break; 2785 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2786 auth_type = SNOW3G_UIA2; 2787 zsk_flag = ZS_IA; 2788 break; 2789 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2790 auth_type = ZUC_EIA3; 2791 zsk_flag = ZS_IA; 2792 break; 2793 case RTE_CRYPTO_AUTH_NULL: 2794 auth_type = 0; 2795 is_null = 1; 2796 break; 2797 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2798 case RTE_CRYPTO_AUTH_AES_CMAC: 2799 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2800 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2801 a_form->algo); 2802 return -1; 2803 default: 2804 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2805 a_form->algo); 2806 return -1; 2807 } 2808 2809 sess->zsk_flag = zsk_flag; 2810 sess->aes_gcm = aes_gcm; 2811 sess->mac_len = a_form->digest_length; 2812 sess->is_null = is_null; 2813 if (zsk_flag) { 2814 sess->auth_iv_offset = a_form->iv.offset; 2815 sess->auth_iv_length = a_form->iv.length; 2816 } 2817 cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data, 2818 a_form->key.length, a_form->digest_length); 2819 2820 return 0; 2821 } 2822 2823 static __rte_always_inline int 2824 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2825 struct cpt_sess_misc *sess) 2826 { 2827 struct rte_crypto_auth_xform *a_form; 2828 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2829 auth_type_t auth_type = 0; /* NULL Auth type */ 2830 void *ctx = SESS_PRIV(sess); 2831 2832 a_form = &xform->auth; 2833 2834 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2835 sess->cpt_op |= CPT_OP_ENCODE; 2836 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2837 sess->cpt_op |= CPT_OP_DECODE; 2838 else { 2839 CPT_LOG_DP_ERR("Unknown auth operation"); 2840 return -1; 2841 } 2842 2843 switch (a_form->algo) { 2844 case RTE_CRYPTO_AUTH_AES_GMAC: 2845 enc_type = AES_GCM; 2846 auth_type = GMAC_TYPE; 2847 break; 2848 default: 2849 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2850 a_form->algo); 2851 return -1; 2852 } 2853 2854 sess->zsk_flag = 0; 2855 sess->aes_gcm = 0; 2856 sess->is_gmac = 1; 2857 sess->iv_offset = a_form->iv.offset; 2858 sess->iv_length = a_form->iv.length; 2859 sess->mac_len = a_form->digest_length; 2860 2861 cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2862 a_form->key.length, NULL); 2863 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length); 2864 2865 return 0; 2866 } 2867 2868 static __rte_always_inline void * 2869 alloc_op_meta(struct rte_mbuf *m_src, 2870 buf_ptr_t *buf, 2871 int32_t len, 2872 struct rte_mempool *cpt_meta_pool) 2873 { 2874 uint8_t *mdata; 2875 2876 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2877 if (likely(m_src && (m_src->nb_segs == 1))) { 2878 int32_t tailroom; 2879 phys_addr_t mphys; 2880 2881 /* Check if tailroom is sufficient to hold meta data */ 2882 tailroom = rte_pktmbuf_tailroom(m_src); 2883 if (likely(tailroom > len + 8)) { 2884 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2885 mphys = m_src->buf_physaddr + m_src->buf_len; 2886 mdata -= len; 2887 mphys -= len; 2888 buf->vaddr = mdata; 2889 buf->dma_addr = mphys; 2890 buf->size = len; 2891 /* Indicate that this is a mbuf allocated mdata */ 2892 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2893 return mdata; 2894 } 2895 } 2896 #else 2897 RTE_SET_USED(m_src); 2898 #endif 2899 2900 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2901 return NULL; 2902 2903 buf->vaddr = mdata; 2904 buf->dma_addr = rte_mempool_virt2iova(mdata); 2905 buf->size = len; 2906 2907 return mdata; 2908 } 2909 2910 /** 2911 * cpt_free_metabuf - free metabuf to mempool. 2912 * @param instance: pointer to instance. 2913 * @param objp: pointer to the metabuf. 2914 */ 2915 static __rte_always_inline void 2916 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2917 { 2918 bool nofree = ((uintptr_t)mdata & 1ull); 2919 2920 if (likely(nofree)) 2921 return; 2922 rte_mempool_put(cpt_meta_pool, mdata); 2923 } 2924 2925 static __rte_always_inline uint32_t 2926 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2927 iov_ptr_t *iovec, uint32_t start_offset) 2928 { 2929 uint16_t index = 0; 2930 void *seg_data = NULL; 2931 phys_addr_t seg_phys; 2932 int32_t seg_size = 0; 2933 2934 if (!pkt) { 2935 iovec->buf_cnt = 0; 2936 return 0; 2937 } 2938 2939 if (!start_offset) { 2940 seg_data = rte_pktmbuf_mtod(pkt, void *); 2941 seg_phys = rte_pktmbuf_mtophys(pkt); 2942 seg_size = pkt->data_len; 2943 } else { 2944 while (start_offset >= pkt->data_len) { 2945 start_offset -= pkt->data_len; 2946 pkt = pkt->next; 2947 } 2948 2949 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2950 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2951 seg_size = pkt->data_len - start_offset; 2952 if (!seg_size) 2953 return 1; 2954 } 2955 2956 /* first seg */ 2957 iovec->bufs[index].vaddr = seg_data; 2958 iovec->bufs[index].dma_addr = seg_phys; 2959 iovec->bufs[index].size = seg_size; 2960 index++; 2961 pkt = pkt->next; 2962 2963 while (unlikely(pkt != NULL)) { 2964 seg_data = rte_pktmbuf_mtod(pkt, void *); 2965 seg_phys = rte_pktmbuf_mtophys(pkt); 2966 seg_size = pkt->data_len; 2967 if (!seg_size) 2968 break; 2969 2970 iovec->bufs[index].vaddr = seg_data; 2971 iovec->bufs[index].dma_addr = seg_phys; 2972 iovec->bufs[index].size = seg_size; 2973 2974 index++; 2975 2976 pkt = pkt->next; 2977 } 2978 2979 iovec->buf_cnt = index; 2980 return 0; 2981 } 2982 2983 static __rte_always_inline uint32_t 2984 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2985 fc_params_t *param, 2986 uint32_t *flags) 2987 { 2988 uint16_t index = 0; 2989 void *seg_data = NULL; 2990 phys_addr_t seg_phys; 2991 uint32_t seg_size = 0; 2992 iov_ptr_t *iovec; 2993 2994 seg_data = rte_pktmbuf_mtod(pkt, void *); 2995 seg_phys = rte_pktmbuf_mtophys(pkt); 2996 seg_size = pkt->data_len; 2997 2998 /* first seg */ 2999 if (likely(!pkt->next)) { 3000 uint32_t headroom, tailroom; 3001 3002 *flags |= SINGLE_BUF_INPLACE; 3003 headroom = rte_pktmbuf_headroom(pkt); 3004 tailroom = rte_pktmbuf_tailroom(pkt); 3005 if (likely((headroom >= 24) && 3006 (tailroom >= 8))) { 3007 /* In 83XX this is prerequivisit for Direct mode */ 3008 *flags |= SINGLE_BUF_HEADTAILROOM; 3009 } 3010 param->bufs[0].vaddr = seg_data; 3011 param->bufs[0].dma_addr = seg_phys; 3012 param->bufs[0].size = seg_size; 3013 return 0; 3014 } 3015 iovec = param->src_iov; 3016 iovec->bufs[index].vaddr = seg_data; 3017 iovec->bufs[index].dma_addr = seg_phys; 3018 iovec->bufs[index].size = seg_size; 3019 index++; 3020 pkt = pkt->next; 3021 3022 while (unlikely(pkt != NULL)) { 3023 seg_data = rte_pktmbuf_mtod(pkt, void *); 3024 seg_phys = rte_pktmbuf_mtophys(pkt); 3025 seg_size = pkt->data_len; 3026 3027 if (!seg_size) 3028 break; 3029 3030 iovec->bufs[index].vaddr = seg_data; 3031 iovec->bufs[index].dma_addr = seg_phys; 3032 iovec->bufs[index].size = seg_size; 3033 3034 index++; 3035 3036 pkt = pkt->next; 3037 } 3038 3039 iovec->buf_cnt = index; 3040 return 0; 3041 } 3042 3043 static __rte_always_inline int 3044 fill_fc_params(struct rte_crypto_op *cop, 3045 struct cpt_sess_misc *sess_misc, 3046 struct cpt_qp_meta_info *m_info, 3047 void **mdata_ptr, 3048 void **prep_req) 3049 { 3050 uint32_t space = 0; 3051 struct rte_crypto_sym_op *sym_op = cop->sym; 3052 void *mdata = NULL; 3053 uintptr_t *op; 3054 uint32_t mc_hash_off; 3055 uint32_t flags = 0; 3056 uint64_t d_offs, d_lens; 3057 struct rte_mbuf *m_src, *m_dst; 3058 uint8_t cpt_op = sess_misc->cpt_op; 3059 #ifdef CPT_ALWAYS_USE_SG_MODE 3060 uint8_t inplace = 0; 3061 #else 3062 uint8_t inplace = 1; 3063 #endif 3064 fc_params_t fc_params; 3065 char src[SRC_IOV_SIZE]; 3066 char dst[SRC_IOV_SIZE]; 3067 uint32_t iv_buf[4]; 3068 int ret; 3069 3070 if (likely(sess_misc->iv_length)) { 3071 flags |= VALID_IV_BUF; 3072 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3073 uint8_t *, sess_misc->iv_offset); 3074 if (sess_misc->aes_ctr && 3075 unlikely(sess_misc->iv_length != 16)) { 3076 memcpy((uint8_t *)iv_buf, 3077 rte_crypto_op_ctod_offset(cop, 3078 uint8_t *, sess_misc->iv_offset), 12); 3079 iv_buf[3] = rte_cpu_to_be_32(0x1); 3080 fc_params.iv_buf = iv_buf; 3081 } 3082 } 3083 3084 if (sess_misc->zsk_flag) { 3085 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3086 uint8_t *, 3087 sess_misc->auth_iv_offset); 3088 if (sess_misc->zsk_flag != ZS_EA) 3089 inplace = 0; 3090 } 3091 m_src = sym_op->m_src; 3092 m_dst = sym_op->m_dst; 3093 3094 if (sess_misc->aes_gcm) { 3095 uint8_t *salt; 3096 uint8_t *aad_data; 3097 uint16_t aad_len; 3098 3099 d_offs = sym_op->aead.data.offset; 3100 d_lens = sym_op->aead.data.length; 3101 mc_hash_off = sym_op->aead.data.offset + 3102 sym_op->aead.data.length; 3103 3104 aad_data = sym_op->aead.aad.data; 3105 aad_len = sess_misc->aad_length; 3106 if (likely((aad_data + aad_len) == 3107 rte_pktmbuf_mtod_offset(m_src, 3108 uint8_t *, 3109 sym_op->aead.data.offset))) { 3110 d_offs = (d_offs - aad_len) | (d_offs << 16); 3111 d_lens = (d_lens + aad_len) | (d_lens << 32); 3112 } else { 3113 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3114 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3115 fc_params.aad_buf.size = aad_len; 3116 flags |= VALID_AAD_BUF; 3117 inplace = 0; 3118 d_offs = d_offs << 16; 3119 d_lens = d_lens << 32; 3120 } 3121 3122 salt = fc_params.iv_buf; 3123 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3124 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3125 sess_misc->salt = *(uint32_t *)salt; 3126 } 3127 fc_params.iv_buf = salt + 4; 3128 if (likely(sess_misc->mac_len)) { 3129 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3130 m_src; 3131 3132 if (!m) 3133 m = m_src; 3134 3135 /* hmac immediately following data is best case */ 3136 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3137 mc_hash_off != 3138 (uint8_t *)sym_op->aead.digest.data)) { 3139 flags |= VALID_MAC_BUF; 3140 fc_params.mac_buf.size = sess_misc->mac_len; 3141 fc_params.mac_buf.vaddr = 3142 sym_op->aead.digest.data; 3143 fc_params.mac_buf.dma_addr = 3144 sym_op->aead.digest.phys_addr; 3145 inplace = 0; 3146 } 3147 } 3148 } else { 3149 d_offs = sym_op->cipher.data.offset; 3150 d_lens = sym_op->cipher.data.length; 3151 mc_hash_off = sym_op->cipher.data.offset + 3152 sym_op->cipher.data.length; 3153 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3154 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3155 3156 if (mc_hash_off < (sym_op->auth.data.offset + 3157 sym_op->auth.data.length)){ 3158 mc_hash_off = (sym_op->auth.data.offset + 3159 sym_op->auth.data.length); 3160 } 3161 /* for gmac, salt should be updated like in gcm */ 3162 if (unlikely(sess_misc->is_gmac)) { 3163 uint8_t *salt; 3164 salt = fc_params.iv_buf; 3165 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3166 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3167 sess_misc->salt = *(uint32_t *)salt; 3168 } 3169 fc_params.iv_buf = salt + 4; 3170 } 3171 if (likely(sess_misc->mac_len)) { 3172 struct rte_mbuf *m; 3173 3174 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3175 if (!m) 3176 m = m_src; 3177 3178 /* hmac immediately following data is best case */ 3179 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3180 mc_hash_off != 3181 (uint8_t *)sym_op->auth.digest.data)) { 3182 flags |= VALID_MAC_BUF; 3183 fc_params.mac_buf.size = 3184 sess_misc->mac_len; 3185 fc_params.mac_buf.vaddr = 3186 sym_op->auth.digest.data; 3187 fc_params.mac_buf.dma_addr = 3188 sym_op->auth.digest.phys_addr; 3189 inplace = 0; 3190 } 3191 } 3192 } 3193 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3194 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3195 3196 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3197 inplace = 0; 3198 3199 if (likely(!m_dst && inplace)) { 3200 /* Case of single buffer without AAD buf or 3201 * separate mac buf in place and 3202 * not air crypto 3203 */ 3204 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3205 3206 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3207 &fc_params, 3208 &flags))) { 3209 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3210 ret = -EINVAL; 3211 goto err_exit; 3212 } 3213 3214 } else { 3215 /* Out of place processing */ 3216 fc_params.src_iov = (void *)src; 3217 fc_params.dst_iov = (void *)dst; 3218 3219 /* Store SG I/O in the api for reuse */ 3220 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3221 CPT_LOG_DP_ERR("Prepare src iov failed"); 3222 ret = -EINVAL; 3223 goto err_exit; 3224 } 3225 3226 if (unlikely(m_dst != NULL)) { 3227 uint32_t pkt_len; 3228 3229 /* Try to make room as much as src has */ 3230 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3231 3232 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3233 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3234 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3235 CPT_LOG_DP_ERR("Not enough space in " 3236 "m_dst %p, need %u" 3237 " more", 3238 m_dst, pkt_len); 3239 ret = -EINVAL; 3240 goto err_exit; 3241 } 3242 } 3243 3244 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3245 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3246 "m_dst %p", m_dst); 3247 ret = -EINVAL; 3248 goto err_exit; 3249 } 3250 } else { 3251 fc_params.dst_iov = (void *)src; 3252 } 3253 } 3254 3255 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3256 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3257 m_info->lb_mlen, m_info->pool); 3258 else 3259 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3260 m_info->sg_mlen, m_info->pool); 3261 3262 if (unlikely(mdata == NULL)) { 3263 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3264 ret = -ENOMEM; 3265 goto err_exit; 3266 } 3267 3268 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3269 op[0] = (uintptr_t)mdata; 3270 op[1] = (uintptr_t)cop; 3271 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3272 space += 4 * sizeof(uint64_t); 3273 3274 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3275 fc_params.meta_buf.dma_addr += space; 3276 fc_params.meta_buf.size -= space; 3277 3278 /* Finally prepare the instruction */ 3279 if (cpt_op & CPT_OP_ENCODE) 3280 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3281 &fc_params, op); 3282 else 3283 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3284 &fc_params, op); 3285 3286 if (unlikely(*prep_req == NULL)) { 3287 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3288 ret = -EINVAL; 3289 goto free_mdata_and_exit; 3290 } 3291 3292 *mdata_ptr = mdata; 3293 3294 return 0; 3295 3296 free_mdata_and_exit: 3297 free_op_meta(mdata, m_info->pool); 3298 err_exit: 3299 return ret; 3300 } 3301 3302 static __rte_always_inline void 3303 compl_auth_verify(struct rte_crypto_op *op, 3304 uint8_t *gen_mac, 3305 uint64_t mac_len) 3306 { 3307 uint8_t *mac; 3308 struct rte_crypto_sym_op *sym_op = op->sym; 3309 3310 if (sym_op->auth.digest.data) 3311 mac = sym_op->auth.digest.data; 3312 else 3313 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3314 uint8_t *, 3315 sym_op->auth.data.length + 3316 sym_op->auth.data.offset); 3317 if (!mac) { 3318 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3319 return; 3320 } 3321 3322 if (memcmp(mac, gen_mac, mac_len)) 3323 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3324 else 3325 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3326 } 3327 3328 static __rte_always_inline int 3329 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3330 { 3331 struct rte_crypto_sym_xform *chain; 3332 3333 CPT_PMD_INIT_FUNC_TRACE(); 3334 3335 if (cpt_is_algo_supported(xform)) 3336 goto err; 3337 3338 chain = xform; 3339 while (chain) { 3340 switch (chain->type) { 3341 case RTE_CRYPTO_SYM_XFORM_AEAD: 3342 if (fill_sess_aead(chain, sess)) 3343 goto err; 3344 break; 3345 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3346 if (fill_sess_cipher(chain, sess)) 3347 goto err; 3348 break; 3349 case RTE_CRYPTO_SYM_XFORM_AUTH: 3350 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3351 if (fill_sess_gmac(chain, sess)) 3352 goto err; 3353 } else { 3354 if (fill_sess_auth(chain, sess)) 3355 goto err; 3356 } 3357 break; 3358 default: 3359 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3360 break; 3361 } 3362 chain = chain->next; 3363 } 3364 3365 return 0; 3366 3367 err: 3368 return -1; 3369 } 3370 3371 static __rte_always_inline void 3372 find_kasumif9_direction_and_length(uint8_t *src, 3373 uint32_t counter_num_bytes, 3374 uint32_t *addr_length_in_bits, 3375 uint8_t *addr_direction) 3376 { 3377 uint8_t found = 0; 3378 uint32_t pos; 3379 uint8_t last_byte; 3380 while (!found && counter_num_bytes > 0) { 3381 counter_num_bytes--; 3382 if (src[counter_num_bytes] == 0x00) 3383 continue; 3384 pos = rte_bsf32(src[counter_num_bytes]); 3385 if (pos == 7) { 3386 if (likely(counter_num_bytes > 0)) { 3387 last_byte = src[counter_num_bytes - 1]; 3388 *addr_direction = last_byte & 0x1; 3389 *addr_length_in_bits = counter_num_bytes * 8 3390 - 1; 3391 } 3392 } else { 3393 last_byte = src[counter_num_bytes]; 3394 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3395 *addr_length_in_bits = counter_num_bytes * 8 3396 + (8 - (pos + 2)); 3397 } 3398 found = 1; 3399 } 3400 } 3401 3402 /* 3403 * This handles all auth only except AES_GMAC 3404 */ 3405 static __rte_always_inline int 3406 fill_digest_params(struct rte_crypto_op *cop, 3407 struct cpt_sess_misc *sess, 3408 struct cpt_qp_meta_info *m_info, 3409 void **mdata_ptr, 3410 void **prep_req) 3411 { 3412 uint32_t space = 0; 3413 struct rte_crypto_sym_op *sym_op = cop->sym; 3414 void *mdata; 3415 phys_addr_t mphys; 3416 uint64_t *op; 3417 uint32_t auth_range_off; 3418 uint32_t flags = 0; 3419 uint64_t d_offs = 0, d_lens; 3420 struct rte_mbuf *m_src, *m_dst; 3421 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3422 uint16_t mac_len = sess->mac_len; 3423 fc_params_t params; 3424 char src[SRC_IOV_SIZE]; 3425 uint8_t iv_buf[16]; 3426 int ret; 3427 3428 memset(¶ms, 0, sizeof(fc_params_t)); 3429 3430 m_src = sym_op->m_src; 3431 3432 /* For just digest lets force mempool alloc */ 3433 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3434 m_info->pool); 3435 if (mdata == NULL) { 3436 ret = -ENOMEM; 3437 goto err_exit; 3438 } 3439 3440 mphys = params.meta_buf.dma_addr; 3441 3442 op = mdata; 3443 op[0] = (uintptr_t)mdata; 3444 op[1] = (uintptr_t)cop; 3445 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3446 space += 4 * sizeof(uint64_t); 3447 3448 auth_range_off = sym_op->auth.data.offset; 3449 3450 flags = VALID_MAC_BUF; 3451 params.src_iov = (void *)src; 3452 if (unlikely(sess->zsk_flag)) { 3453 /* 3454 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3455 * we will send pass through even for auth only case, 3456 * let MC handle it 3457 */ 3458 d_offs = auth_range_off; 3459 auth_range_off = 0; 3460 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3461 uint8_t *, sess->auth_iv_offset); 3462 if (sess->zsk_flag == K_F9) { 3463 uint32_t length_in_bits, num_bytes; 3464 uint8_t *src, direction = 0; 3465 3466 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3467 uint8_t *), 8); 3468 /* 3469 * This is kasumi f9, take direction from 3470 * source buffer 3471 */ 3472 length_in_bits = cop->sym->auth.data.length; 3473 num_bytes = (length_in_bits >> 3); 3474 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3475 find_kasumif9_direction_and_length(src, 3476 num_bytes, 3477 &length_in_bits, 3478 &direction); 3479 length_in_bits -= 64; 3480 cop->sym->auth.data.offset += 64; 3481 d_offs = cop->sym->auth.data.offset; 3482 auth_range_off = d_offs / 8; 3483 cop->sym->auth.data.length = length_in_bits; 3484 3485 /* Store it at end of auth iv */ 3486 iv_buf[8] = direction; 3487 params.auth_iv_buf = iv_buf; 3488 } 3489 } 3490 3491 d_lens = sym_op->auth.data.length; 3492 3493 params.ctx_buf.vaddr = SESS_PRIV(sess); 3494 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3495 3496 if (auth_op == CPT_OP_AUTH_GENERATE) { 3497 if (sym_op->auth.digest.data) { 3498 /* 3499 * Digest to be generated 3500 * in separate buffer 3501 */ 3502 params.mac_buf.size = 3503 sess->mac_len; 3504 params.mac_buf.vaddr = 3505 sym_op->auth.digest.data; 3506 params.mac_buf.dma_addr = 3507 sym_op->auth.digest.phys_addr; 3508 } else { 3509 uint32_t off = sym_op->auth.data.offset + 3510 sym_op->auth.data.length; 3511 int32_t dlen, space; 3512 3513 m_dst = sym_op->m_dst ? 3514 sym_op->m_dst : sym_op->m_src; 3515 dlen = rte_pktmbuf_pkt_len(m_dst); 3516 3517 space = off + mac_len - dlen; 3518 if (space > 0) 3519 if (!rte_pktmbuf_append(m_dst, space)) { 3520 CPT_LOG_DP_ERR("Failed to extend " 3521 "mbuf by %uB", space); 3522 ret = -EINVAL; 3523 goto free_mdata_and_exit; 3524 } 3525 3526 params.mac_buf.vaddr = 3527 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3528 params.mac_buf.dma_addr = 3529 rte_pktmbuf_mtophys_offset(m_dst, off); 3530 params.mac_buf.size = mac_len; 3531 } 3532 } else { 3533 /* Need space for storing generated mac */ 3534 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3535 params.mac_buf.dma_addr = mphys + space; 3536 params.mac_buf.size = mac_len; 3537 space += RTE_ALIGN_CEIL(mac_len, 8); 3538 op[2] = (uintptr_t)params.mac_buf.vaddr; 3539 op[3] = mac_len; 3540 } 3541 3542 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3543 params.meta_buf.dma_addr = mphys + space; 3544 params.meta_buf.size -= space; 3545 3546 /* Out of place processing */ 3547 params.src_iov = (void *)src; 3548 3549 /*Store SG I/O in the api for reuse */ 3550 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3551 CPT_LOG_DP_ERR("Prepare src iov failed"); 3552 ret = -EINVAL; 3553 goto free_mdata_and_exit; 3554 } 3555 3556 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3557 if (unlikely(*prep_req == NULL)) { 3558 ret = -EINVAL; 3559 goto free_mdata_and_exit; 3560 } 3561 3562 *mdata_ptr = mdata; 3563 3564 return 0; 3565 3566 free_mdata_and_exit: 3567 free_op_meta(mdata, m_info->pool); 3568 err_exit: 3569 return ret; 3570 } 3571 3572 #endif /*_CPT_UCODE_H_ */ 3573