1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline void 26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 27 { 28 int i, base; 29 30 for (i = 0; i < 4; i++) { 31 base = 4 * i; 32 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 33 (ck[base + 2] << 8) | (ck[base + 3]); 34 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 35 } 36 } 37 38 static __rte_always_inline int 39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) 40 { 41 uint16_t mac_len = auth->digest_length; 42 int ret; 43 44 switch (auth->algo) { 45 case RTE_CRYPTO_AUTH_MD5: 46 case RTE_CRYPTO_AUTH_MD5_HMAC: 47 ret = (mac_len == 16) ? 0 : -1; 48 break; 49 case RTE_CRYPTO_AUTH_SHA1: 50 case RTE_CRYPTO_AUTH_SHA1_HMAC: 51 ret = (mac_len == 20) ? 0 : -1; 52 break; 53 case RTE_CRYPTO_AUTH_SHA224: 54 case RTE_CRYPTO_AUTH_SHA224_HMAC: 55 ret = (mac_len == 28) ? 0 : -1; 56 break; 57 case RTE_CRYPTO_AUTH_SHA256: 58 case RTE_CRYPTO_AUTH_SHA256_HMAC: 59 ret = (mac_len == 32) ? 0 : -1; 60 break; 61 case RTE_CRYPTO_AUTH_SHA384: 62 case RTE_CRYPTO_AUTH_SHA384_HMAC: 63 ret = (mac_len == 48) ? 0 : -1; 64 break; 65 case RTE_CRYPTO_AUTH_SHA512: 66 case RTE_CRYPTO_AUTH_SHA512_HMAC: 67 ret = (mac_len == 64) ? 0 : -1; 68 break; 69 case RTE_CRYPTO_AUTH_NULL: 70 ret = 0; 71 break; 72 default: 73 ret = -1; 74 } 75 76 return ret; 77 } 78 79 static __rte_always_inline void 80 cpt_fc_salt_update(struct cpt_ctx *cpt_ctx, 81 uint8_t *salt) 82 { 83 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 84 memcpy(fctx->enc.encr_iv, salt, 4); 85 } 86 87 static __rte_always_inline int 88 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 89 { 90 switch (key_len) { 91 case 16: 92 case 24: 93 case 32: 94 return 0; 95 default: 96 return -1; 97 } 98 } 99 100 static __rte_always_inline int 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 102 { 103 int fc_type = 0; 104 switch (type) { 105 case PASSTHROUGH: 106 fc_type = FC_GEN; 107 break; 108 case DES3_CBC: 109 case DES3_ECB: 110 fc_type = FC_GEN; 111 break; 112 case AES_CBC: 113 case AES_ECB: 114 case AES_CFB: 115 case AES_CTR: 116 case AES_GCM: 117 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 118 return -1; 119 fc_type = FC_GEN; 120 break; 121 case CHACHA20: 122 fc_type = FC_GEN; 123 break; 124 case AES_XTS: 125 key_len = key_len / 2; 126 if (unlikely(key_len == 24)) { 127 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 128 return -1; 129 } 130 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 131 return -1; 132 fc_type = FC_GEN; 133 break; 134 case ZUC_EEA3: 135 case SNOW3G_UEA2: 136 if (unlikely(key_len != 16)) 137 return -1; 138 /* No support for AEAD yet */ 139 if (unlikely(ctx->hash_type)) 140 return -1; 141 fc_type = ZUC_SNOW3G; 142 break; 143 case KASUMI_F8_CBC: 144 case KASUMI_F8_ECB: 145 if (unlikely(key_len != 16)) 146 return -1; 147 /* No support for AEAD yet */ 148 if (unlikely(ctx->hash_type)) 149 return -1; 150 fc_type = KASUMI; 151 break; 152 default: 153 return -1; 154 } 155 156 ctx->fc_type = fc_type; 157 return 0; 158 } 159 160 static __rte_always_inline void 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 162 { 163 cpt_ctx->enc_cipher = 0; 164 fctx->enc.enc_cipher = 0; 165 } 166 167 static __rte_always_inline void 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 169 { 170 mc_aes_type_t aes_key_type = 0; 171 switch (key_len) { 172 case 16: 173 aes_key_type = AES_128_BIT; 174 break; 175 case 24: 176 aes_key_type = AES_192_BIT; 177 break; 178 case 32: 179 aes_key_type = AES_256_BIT; 180 break; 181 default: 182 /* This should not happen */ 183 CPT_LOG_DP_ERR("Invalid AES key len"); 184 return; 185 } 186 fctx->enc.aes_key = aes_key_type; 187 } 188 189 static __rte_always_inline void 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 191 uint16_t key_len) 192 { 193 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 194 uint32_t keyx[4]; 195 196 cpt_ctx->snow3g = 1; 197 gen_key_snow3g(key, keyx); 198 memcpy(zs_ctx->ci_key, keyx, key_len); 199 cpt_ctx->zsk_flags = 0; 200 } 201 202 static __rte_always_inline void 203 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 204 uint16_t key_len) 205 { 206 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 207 208 cpt_ctx->snow3g = 0; 209 memcpy(zs_ctx->ci_key, key, key_len); 210 memcpy(zs_ctx->zuc_const, zuc_d, 32); 211 cpt_ctx->zsk_flags = 0; 212 } 213 214 static __rte_always_inline void 215 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 216 uint16_t key_len) 217 { 218 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 219 220 cpt_ctx->k_ecb = 1; 221 memcpy(k_ctx->ci_key, key, key_len); 222 cpt_ctx->zsk_flags = 0; 223 } 224 225 static __rte_always_inline void 226 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 227 uint16_t key_len) 228 { 229 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 230 231 memcpy(k_ctx->ci_key, key, key_len); 232 cpt_ctx->zsk_flags = 0; 233 } 234 235 static __rte_always_inline int 236 cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type, 237 const uint8_t *key, uint16_t key_len, uint8_t *salt) 238 { 239 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 240 int ret; 241 242 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 243 if (unlikely(ret)) 244 return -1; 245 246 if (cpt_ctx->fc_type == FC_GEN) { 247 /* 248 * We need to always say IV is from DPTR as user can 249 * sometimes override IV per operation. 250 */ 251 fctx->enc.iv_source = CPT_FROM_DPTR; 252 253 if (cpt_ctx->auth_key_len > 64) 254 return -1; 255 } 256 257 switch (type) { 258 case PASSTHROUGH: 259 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 260 goto success; 261 case DES3_CBC: 262 /* CPT performs DES using 3DES with the 8B DES-key 263 * replicated 2 more times to match the 24B 3DES-key. 264 * Eg. If org. key is "0x0a 0x0b", then new key is 265 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 266 */ 267 if (key_len == 8) { 268 /* Skipping the first 8B as it will be copied 269 * in the regular code flow 270 */ 271 memcpy(fctx->enc.encr_key+key_len, key, key_len); 272 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 273 } 274 break; 275 case DES3_ECB: 276 /* For DES3_ECB IV need to be from CTX. */ 277 fctx->enc.iv_source = CPT_FROM_CTX; 278 break; 279 case AES_CBC: 280 case AES_ECB: 281 case AES_CFB: 282 case AES_CTR: 283 case CHACHA20: 284 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 285 break; 286 case AES_GCM: 287 /* Even though iv source is from dptr, 288 * aes_gcm salt is taken from ctx 289 */ 290 if (salt) { 291 memcpy(fctx->enc.encr_iv, salt, 4); 292 /* Assuming it was just salt update 293 * and nothing else 294 */ 295 if (!key) 296 goto success; 297 } 298 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 299 break; 300 case AES_XTS: 301 key_len = key_len / 2; 302 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 303 304 /* Copy key2 for XTS into ipad */ 305 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 306 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 307 break; 308 case SNOW3G_UEA2: 309 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 310 goto success; 311 case ZUC_EEA3: 312 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 313 goto success; 314 case KASUMI_F8_ECB: 315 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 316 goto success; 317 case KASUMI_F8_CBC: 318 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 319 goto success; 320 default: 321 return -1; 322 } 323 324 /* Only for FC_GEN case */ 325 326 /* For GMAC auth, cipher must be NULL */ 327 if (cpt_ctx->hash_type != GMAC_TYPE) 328 fctx->enc.enc_cipher = type; 329 330 memcpy(fctx->enc.encr_key, key, key_len); 331 332 success: 333 cpt_ctx->enc_cipher = type; 334 335 return 0; 336 } 337 338 static __rte_always_inline uint32_t 339 fill_sg_comp(sg_comp_t *list, 340 uint32_t i, 341 phys_addr_t dma_addr, 342 uint32_t size) 343 { 344 sg_comp_t *to = &list[i>>2]; 345 346 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 347 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 348 i++; 349 return i; 350 } 351 352 static __rte_always_inline uint32_t 353 fill_sg_comp_from_buf(sg_comp_t *list, 354 uint32_t i, 355 buf_ptr_t *from) 356 { 357 sg_comp_t *to = &list[i>>2]; 358 359 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 360 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 361 i++; 362 return i; 363 } 364 365 static __rte_always_inline uint32_t 366 fill_sg_comp_from_buf_min(sg_comp_t *list, 367 uint32_t i, 368 buf_ptr_t *from, 369 uint32_t *psize) 370 { 371 sg_comp_t *to = &list[i >> 2]; 372 uint32_t size = *psize; 373 uint32_t e_len; 374 375 e_len = (size > from->size) ? from->size : size; 376 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 377 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 378 *psize -= e_len; 379 i++; 380 return i; 381 } 382 383 /* 384 * This fills the MC expected SGIO list 385 * from IOV given by user. 386 */ 387 static __rte_always_inline uint32_t 388 fill_sg_comp_from_iov(sg_comp_t *list, 389 uint32_t i, 390 iov_ptr_t *from, uint32_t from_offset, 391 uint32_t *psize, buf_ptr_t *extra_buf, 392 uint32_t extra_offset) 393 { 394 int32_t j; 395 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 396 uint32_t size = *psize; 397 398 for (j = 0; (j < from->buf_cnt) && size; j++) { 399 phys_addr_t dma_addr = from->bufs[j].dma_addr; 400 uint32_t buf_sz = from->bufs[j].size; 401 sg_comp_t *to = &list[i >> 2]; 402 phys_addr_t e_dma_addr; 403 uint32_t e_len; 404 405 if (unlikely(from_offset)) { 406 if (from_offset >= buf_sz) { 407 from_offset -= buf_sz; 408 continue; 409 } 410 e_dma_addr = dma_addr + from_offset; 411 e_len = (size > (buf_sz - from_offset)) ? 412 (buf_sz - from_offset) : size; 413 from_offset = 0; 414 } else { 415 e_dma_addr = dma_addr; 416 e_len = (size > buf_sz) ? buf_sz : size; 417 } 418 419 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 420 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 421 422 if (extra_len && (e_len >= extra_offset)) { 423 /* Break the data at given offset */ 424 uint32_t next_len = e_len - extra_offset; 425 phys_addr_t next_dma = e_dma_addr + extra_offset; 426 427 if (!extra_offset) { 428 i--; 429 } else { 430 e_len = extra_offset; 431 size -= e_len; 432 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 433 } 434 435 extra_len = RTE_MIN(extra_len, size); 436 /* Insert extra data ptr */ 437 if (extra_len) { 438 i++; 439 to = &list[i >> 2]; 440 to->u.s.len[i % 4] = 441 rte_cpu_to_be_16(extra_len); 442 to->ptr[i % 4] = 443 rte_cpu_to_be_64(extra_buf->dma_addr); 444 size -= extra_len; 445 } 446 447 next_len = RTE_MIN(next_len, size); 448 /* insert the rest of the data */ 449 if (next_len) { 450 i++; 451 to = &list[i >> 2]; 452 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 453 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 454 size -= next_len; 455 } 456 extra_len = 0; 457 458 } else { 459 size -= e_len; 460 } 461 if (extra_offset) 462 extra_offset -= size; 463 i++; 464 } 465 466 *psize = size; 467 return (uint32_t)i; 468 } 469 470 static __rte_always_inline void 471 cpt_digest_gen_prep(uint32_t flags, 472 uint64_t d_lens, 473 digest_params_t *params, 474 void *op, 475 void **prep_req) 476 { 477 struct cpt_request_info *req; 478 uint32_t size, i; 479 uint16_t data_len, mac_len, key_len; 480 auth_type_t hash_type; 481 buf_ptr_t *meta_p; 482 struct cpt_ctx *ctx; 483 sg_comp_t *gather_comp; 484 sg_comp_t *scatter_comp; 485 uint8_t *in_buffer; 486 uint32_t g_size_bytes, s_size_bytes; 487 uint64_t dptr_dma, rptr_dma; 488 vq_cmd_word0_t vq_cmd_w0; 489 void *c_vaddr, *m_vaddr; 490 uint64_t c_dma, m_dma; 491 492 ctx = params->ctx_buf.vaddr; 493 meta_p = ¶ms->meta_buf; 494 495 m_vaddr = meta_p->vaddr; 496 m_dma = meta_p->dma_addr; 497 498 /* 499 * Save initial space that followed app data for completion code & 500 * alternate completion code to fall in same cache line as app data 501 */ 502 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 503 m_dma += COMPLETION_CODE_SIZE; 504 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 505 (uint8_t *)m_vaddr; 506 c_vaddr = (uint8_t *)m_vaddr + size; 507 c_dma = m_dma + size; 508 size += sizeof(cpt_res_s_t); 509 510 m_vaddr = (uint8_t *)m_vaddr + size; 511 m_dma += size; 512 513 req = m_vaddr; 514 515 size = sizeof(struct cpt_request_info); 516 m_vaddr = (uint8_t *)m_vaddr + size; 517 m_dma += size; 518 519 hash_type = ctx->hash_type; 520 mac_len = ctx->mac_len; 521 key_len = ctx->auth_key_len; 522 data_len = AUTH_DLEN(d_lens); 523 524 /*GP op header */ 525 vq_cmd_w0.s.opcode.minor = 0; 526 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 527 if (ctx->hmac) { 528 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 529 vq_cmd_w0.s.param1 = key_len; 530 vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8); 531 } else { 532 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 533 vq_cmd_w0.s.param1 = 0; 534 vq_cmd_w0.s.dlen = data_len; 535 } 536 537 /* Null auth only case enters the if */ 538 if (unlikely(!hash_type && !ctx->enc_cipher)) { 539 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC; 540 /* Minor op is passthrough */ 541 vq_cmd_w0.s.opcode.minor = 0x03; 542 /* Send out completion code only */ 543 vq_cmd_w0.s.param2 = 0x1; 544 } 545 546 /* DPTR has SG list */ 547 in_buffer = m_vaddr; 548 dptr_dma = m_dma; 549 550 ((uint16_t *)in_buffer)[0] = 0; 551 ((uint16_t *)in_buffer)[1] = 0; 552 553 /* TODO Add error check if space will be sufficient */ 554 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 555 556 /* 557 * Input gather list 558 */ 559 560 i = 0; 561 562 if (ctx->hmac) { 563 uint64_t k_dma = ctx->auth_key_iova; 564 /* Key */ 565 i = fill_sg_comp(gather_comp, i, k_dma, 566 RTE_ALIGN_CEIL(key_len, 8)); 567 } 568 569 /* input data */ 570 size = data_len; 571 if (size) { 572 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 573 0, &size, NULL, 0); 574 if (unlikely(size)) { 575 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 576 " by %dB", size); 577 return; 578 } 579 } else { 580 /* 581 * Looks like we need to support zero data 582 * gather ptr in case of hash & hmac 583 */ 584 i++; 585 } 586 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 587 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 588 589 /* 590 * Output Gather list 591 */ 592 593 i = 0; 594 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 595 596 if (flags & VALID_MAC_BUF) { 597 if (unlikely(params->mac_buf.size < mac_len)) { 598 CPT_LOG_DP_ERR("Insufficient MAC size"); 599 return; 600 } 601 602 size = mac_len; 603 i = fill_sg_comp_from_buf_min(scatter_comp, i, 604 ¶ms->mac_buf, &size); 605 } else { 606 size = mac_len; 607 i = fill_sg_comp_from_iov(scatter_comp, i, 608 params->src_iov, data_len, 609 &size, NULL, 0); 610 if (unlikely(size)) { 611 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 612 " %dB", size); 613 return; 614 } 615 } 616 617 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 618 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 619 620 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 621 622 /* This is DPTR len incase of SG mode */ 623 vq_cmd_w0.s.dlen = size; 624 625 m_vaddr = (uint8_t *)m_vaddr + size; 626 m_dma += size; 627 628 /* cpt alternate completion address saved earlier */ 629 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 630 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 631 rptr_dma = c_dma - 8; 632 633 req->ist.ei1 = dptr_dma; 634 req->ist.ei2 = rptr_dma; 635 636 /* 16 byte aligned cpt res address */ 637 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 638 *req->completion_addr = COMPLETION_CODE_INIT; 639 req->comp_baddr = c_dma; 640 641 /* Fill microcode part of instruction */ 642 req->ist.ei0 = vq_cmd_w0.u64; 643 644 req->op = op; 645 646 *prep_req = req; 647 return; 648 } 649 650 static __rte_always_inline void 651 cpt_enc_hmac_prep(uint32_t flags, 652 uint64_t d_offs, 653 uint64_t d_lens, 654 fc_params_t *fc_params, 655 void *op, 656 void **prep_req) 657 { 658 uint32_t iv_offset = 0; 659 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 660 struct cpt_ctx *cpt_ctx; 661 uint32_t cipher_type, hash_type; 662 uint32_t mac_len, size; 663 uint8_t iv_len = 16; 664 struct cpt_request_info *req; 665 buf_ptr_t *meta_p, *aad_buf = NULL; 666 uint32_t encr_offset, auth_offset; 667 uint32_t encr_data_len, auth_data_len, aad_len = 0; 668 uint32_t passthrough_len = 0; 669 void *m_vaddr, *offset_vaddr; 670 uint64_t m_dma, offset_dma; 671 vq_cmd_word0_t vq_cmd_w0; 672 void *c_vaddr; 673 uint64_t c_dma; 674 675 meta_p = &fc_params->meta_buf; 676 m_vaddr = meta_p->vaddr; 677 m_dma = meta_p->dma_addr; 678 679 encr_offset = ENCR_OFFSET(d_offs); 680 auth_offset = AUTH_OFFSET(d_offs); 681 encr_data_len = ENCR_DLEN(d_lens); 682 auth_data_len = AUTH_DLEN(d_lens); 683 if (unlikely(flags & VALID_AAD_BUF)) { 684 /* 685 * We dont support both aad 686 * and auth data separately 687 */ 688 auth_data_len = 0; 689 auth_offset = 0; 690 aad_len = fc_params->aad_buf.size; 691 aad_buf = &fc_params->aad_buf; 692 } 693 cpt_ctx = fc_params->ctx_buf.vaddr; 694 cipher_type = cpt_ctx->enc_cipher; 695 hash_type = cpt_ctx->hash_type; 696 mac_len = cpt_ctx->mac_len; 697 698 /* 699 * Save initial space that followed app data for completion code & 700 * alternate completion code to fall in same cache line as app data 701 */ 702 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 703 m_dma += COMPLETION_CODE_SIZE; 704 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 705 (uint8_t *)m_vaddr; 706 707 c_vaddr = (uint8_t *)m_vaddr + size; 708 c_dma = m_dma + size; 709 size += sizeof(cpt_res_s_t); 710 711 m_vaddr = (uint8_t *)m_vaddr + size; 712 m_dma += size; 713 714 /* start cpt request info struct at 8 byte boundary */ 715 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 716 (uint8_t *)m_vaddr; 717 718 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 719 720 size += sizeof(struct cpt_request_info); 721 m_vaddr = (uint8_t *)m_vaddr + size; 722 m_dma += size; 723 724 if (unlikely(!(flags & VALID_IV_BUF))) { 725 iv_len = 0; 726 iv_offset = ENCR_IV_OFFSET(d_offs); 727 } 728 729 if (unlikely(flags & VALID_AAD_BUF)) { 730 /* 731 * When AAD is given, data above encr_offset is pass through 732 * Since AAD is given as separate pointer and not as offset, 733 * this is a special case as we need to fragment input data 734 * into passthrough + encr_data and then insert AAD in between. 735 */ 736 if (hash_type != GMAC_TYPE) { 737 passthrough_len = encr_offset; 738 auth_offset = passthrough_len + iv_len; 739 encr_offset = passthrough_len + aad_len + iv_len; 740 auth_data_len = aad_len + encr_data_len; 741 } else { 742 passthrough_len = 16 + aad_len; 743 auth_offset = passthrough_len + iv_len; 744 auth_data_len = aad_len; 745 } 746 } else { 747 encr_offset += iv_len; 748 auth_offset += iv_len; 749 } 750 751 /* Encryption */ 752 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 753 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT; 754 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc << 755 CPT_HMAC_FIRST_BIT_POS); 756 757 if (hash_type == GMAC_TYPE) { 758 encr_offset = 0; 759 encr_data_len = 0; 760 } 761 762 auth_dlen = auth_offset + auth_data_len; 763 enc_dlen = encr_data_len + encr_offset; 764 if (unlikely(encr_data_len & 0xf)) { 765 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 766 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 767 encr_offset; 768 else if (likely((cipher_type == AES_CBC) || 769 (cipher_type == AES_ECB))) 770 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 771 encr_offset; 772 } 773 774 if (unlikely(auth_dlen > enc_dlen)) { 775 inputlen = auth_dlen; 776 outputlen = auth_dlen + mac_len; 777 } else { 778 inputlen = enc_dlen; 779 outputlen = enc_dlen + mac_len; 780 } 781 782 if (cpt_ctx->auth_enc != 0) 783 outputlen = enc_dlen; 784 785 /* GP op header */ 786 vq_cmd_w0.s.param1 = encr_data_len; 787 vq_cmd_w0.s.param2 = auth_data_len; 788 /* 789 * In 83XX since we have a limitation of 790 * IV & Offset control word not part of instruction 791 * and need to be part of Data Buffer, we check if 792 * head room is there and then only do the Direct mode processing 793 */ 794 if (likely((flags & SINGLE_BUF_INPLACE) && 795 (flags & SINGLE_BUF_HEADTAILROOM))) { 796 void *dm_vaddr = fc_params->bufs[0].vaddr; 797 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 798 /* 799 * This flag indicates that there is 24 bytes head room and 800 * 8 bytes tail room available, so that we get to do 801 * DIRECT MODE with limitation 802 */ 803 804 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 805 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 806 807 /* DPTR */ 808 req->ist.ei1 = offset_dma; 809 /* RPTR should just exclude offset control word */ 810 req->ist.ei2 = dm_dma_addr - iv_len; 811 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 812 + outputlen - iv_len); 813 814 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 815 816 if (likely(iv_len)) { 817 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 818 + OFF_CTRL_LEN); 819 uint64_t *src = fc_params->iv_buf; 820 dest[0] = src[0]; 821 dest[1] = src[1]; 822 } 823 824 *(uint64_t *)offset_vaddr = 825 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 826 ((uint64_t)iv_offset << 8) | 827 ((uint64_t)auth_offset)); 828 829 } else { 830 uint32_t i, g_size_bytes, s_size_bytes; 831 uint64_t dptr_dma, rptr_dma; 832 sg_comp_t *gather_comp; 833 sg_comp_t *scatter_comp; 834 uint8_t *in_buffer; 835 836 /* This falls under strict SG mode */ 837 offset_vaddr = m_vaddr; 838 offset_dma = m_dma; 839 size = OFF_CTRL_LEN + iv_len; 840 841 m_vaddr = (uint8_t *)m_vaddr + size; 842 m_dma += size; 843 844 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 845 846 if (likely(iv_len)) { 847 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 848 + OFF_CTRL_LEN); 849 uint64_t *src = fc_params->iv_buf; 850 dest[0] = src[0]; 851 dest[1] = src[1]; 852 } 853 854 *(uint64_t *)offset_vaddr = 855 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 856 ((uint64_t)iv_offset << 8) | 857 ((uint64_t)auth_offset)); 858 859 /* DPTR has SG list */ 860 in_buffer = m_vaddr; 861 dptr_dma = m_dma; 862 863 ((uint16_t *)in_buffer)[0] = 0; 864 ((uint16_t *)in_buffer)[1] = 0; 865 866 /* TODO Add error check if space will be sufficient */ 867 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 868 869 /* 870 * Input Gather List 871 */ 872 873 i = 0; 874 875 /* Offset control word that includes iv */ 876 i = fill_sg_comp(gather_comp, i, offset_dma, 877 OFF_CTRL_LEN + iv_len); 878 879 /* Add input data */ 880 size = inputlen - iv_len; 881 if (likely(size)) { 882 uint32_t aad_offset = aad_len ? passthrough_len : 0; 883 884 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 885 i = fill_sg_comp_from_buf_min(gather_comp, i, 886 fc_params->bufs, 887 &size); 888 } else { 889 i = fill_sg_comp_from_iov(gather_comp, i, 890 fc_params->src_iov, 891 0, &size, 892 aad_buf, aad_offset); 893 } 894 895 if (unlikely(size)) { 896 CPT_LOG_DP_ERR("Insufficient buffer space," 897 " size %d needed", size); 898 return; 899 } 900 } 901 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 902 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 903 904 /* 905 * Output Scatter list 906 */ 907 i = 0; 908 scatter_comp = 909 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 910 911 /* Add IV */ 912 if (likely(iv_len)) { 913 i = fill_sg_comp(scatter_comp, i, 914 offset_dma + OFF_CTRL_LEN, 915 iv_len); 916 } 917 918 /* output data or output data + digest*/ 919 if (unlikely(flags & VALID_MAC_BUF)) { 920 size = outputlen - iv_len - mac_len; 921 if (size) { 922 uint32_t aad_offset = 923 aad_len ? passthrough_len : 0; 924 925 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 926 i = fill_sg_comp_from_buf_min( 927 scatter_comp, 928 i, 929 fc_params->bufs, 930 &size); 931 } else { 932 i = fill_sg_comp_from_iov(scatter_comp, 933 i, 934 fc_params->dst_iov, 935 0, 936 &size, 937 aad_buf, 938 aad_offset); 939 } 940 if (unlikely(size)) { 941 CPT_LOG_DP_ERR("Insufficient buffer" 942 " space, size %d needed", 943 size); 944 return; 945 } 946 } 947 /* mac_data */ 948 if (mac_len) { 949 i = fill_sg_comp_from_buf(scatter_comp, i, 950 &fc_params->mac_buf); 951 } 952 } else { 953 /* Output including mac */ 954 size = outputlen - iv_len; 955 if (likely(size)) { 956 uint32_t aad_offset = 957 aad_len ? passthrough_len : 0; 958 959 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 960 i = fill_sg_comp_from_buf_min( 961 scatter_comp, 962 i, 963 fc_params->bufs, 964 &size); 965 } else { 966 i = fill_sg_comp_from_iov(scatter_comp, 967 i, 968 fc_params->dst_iov, 969 0, 970 &size, 971 aad_buf, 972 aad_offset); 973 } 974 if (unlikely(size)) { 975 CPT_LOG_DP_ERR("Insufficient buffer" 976 " space, size %d needed", 977 size); 978 return; 979 } 980 } 981 } 982 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 983 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 984 985 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 986 987 /* This is DPTR len incase of SG mode */ 988 vq_cmd_w0.s.dlen = size; 989 990 m_vaddr = (uint8_t *)m_vaddr + size; 991 m_dma += size; 992 993 /* cpt alternate completion address saved earlier */ 994 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 995 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 996 rptr_dma = c_dma - 8; 997 998 req->ist.ei1 = dptr_dma; 999 req->ist.ei2 = rptr_dma; 1000 } 1001 1002 if (unlikely((encr_offset >> 16) || 1003 (iv_offset >> 8) || 1004 (auth_offset >> 8))) { 1005 CPT_LOG_DP_ERR("Offset not supported"); 1006 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1007 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1008 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1009 return; 1010 } 1011 1012 /* 16 byte aligned cpt res address */ 1013 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1014 *req->completion_addr = COMPLETION_CODE_INIT; 1015 req->comp_baddr = c_dma; 1016 1017 /* Fill microcode part of instruction */ 1018 req->ist.ei0 = vq_cmd_w0.u64; 1019 1020 req->op = op; 1021 1022 *prep_req = req; 1023 return; 1024 } 1025 1026 static __rte_always_inline void 1027 cpt_dec_hmac_prep(uint32_t flags, 1028 uint64_t d_offs, 1029 uint64_t d_lens, 1030 fc_params_t *fc_params, 1031 void *op, 1032 void **prep_req) 1033 { 1034 uint32_t iv_offset = 0, size; 1035 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1036 struct cpt_ctx *cpt_ctx; 1037 int32_t hash_type, mac_len; 1038 uint8_t iv_len = 16; 1039 struct cpt_request_info *req; 1040 buf_ptr_t *meta_p, *aad_buf = NULL; 1041 uint32_t encr_offset, auth_offset; 1042 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1043 uint32_t passthrough_len = 0; 1044 void *m_vaddr, *offset_vaddr; 1045 uint64_t m_dma, offset_dma; 1046 vq_cmd_word0_t vq_cmd_w0; 1047 void *c_vaddr; 1048 uint64_t c_dma; 1049 1050 meta_p = &fc_params->meta_buf; 1051 m_vaddr = meta_p->vaddr; 1052 m_dma = meta_p->dma_addr; 1053 1054 encr_offset = ENCR_OFFSET(d_offs); 1055 auth_offset = AUTH_OFFSET(d_offs); 1056 encr_data_len = ENCR_DLEN(d_lens); 1057 auth_data_len = AUTH_DLEN(d_lens); 1058 1059 if (unlikely(flags & VALID_AAD_BUF)) { 1060 /* 1061 * We dont support both aad 1062 * and auth data separately 1063 */ 1064 auth_data_len = 0; 1065 auth_offset = 0; 1066 aad_len = fc_params->aad_buf.size; 1067 aad_buf = &fc_params->aad_buf; 1068 } 1069 1070 cpt_ctx = fc_params->ctx_buf.vaddr; 1071 hash_type = cpt_ctx->hash_type; 1072 mac_len = cpt_ctx->mac_len; 1073 1074 if (unlikely(!(flags & VALID_IV_BUF))) { 1075 iv_len = 0; 1076 iv_offset = ENCR_IV_OFFSET(d_offs); 1077 } 1078 1079 if (unlikely(flags & VALID_AAD_BUF)) { 1080 /* 1081 * When AAD is given, data above encr_offset is pass through 1082 * Since AAD is given as separate pointer and not as offset, 1083 * this is a special case as we need to fragment input data 1084 * into passthrough + encr_data and then insert AAD in between. 1085 */ 1086 if (hash_type != GMAC_TYPE) { 1087 passthrough_len = encr_offset; 1088 auth_offset = passthrough_len + iv_len; 1089 encr_offset = passthrough_len + aad_len + iv_len; 1090 auth_data_len = aad_len + encr_data_len; 1091 } else { 1092 passthrough_len = 16 + aad_len; 1093 auth_offset = passthrough_len + iv_len; 1094 auth_data_len = aad_len; 1095 } 1096 } else { 1097 encr_offset += iv_len; 1098 auth_offset += iv_len; 1099 } 1100 1101 /* 1102 * Save initial space that followed app data for completion code & 1103 * alternate completion code to fall in same cache line as app data 1104 */ 1105 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1106 m_dma += COMPLETION_CODE_SIZE; 1107 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1108 (uint8_t *)m_vaddr; 1109 c_vaddr = (uint8_t *)m_vaddr + size; 1110 c_dma = m_dma + size; 1111 size += sizeof(cpt_res_s_t); 1112 1113 m_vaddr = (uint8_t *)m_vaddr + size; 1114 m_dma += size; 1115 1116 /* start cpt request info structure at 8 byte alignment */ 1117 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1118 (uint8_t *)m_vaddr; 1119 1120 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1121 1122 size += sizeof(struct cpt_request_info); 1123 m_vaddr = (uint8_t *)m_vaddr + size; 1124 m_dma += size; 1125 1126 /* Decryption */ 1127 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 1128 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT; 1129 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth << 1130 CPT_HMAC_FIRST_BIT_POS); 1131 1132 if (hash_type == GMAC_TYPE) { 1133 encr_offset = 0; 1134 encr_data_len = 0; 1135 } 1136 1137 enc_dlen = encr_offset + encr_data_len; 1138 auth_dlen = auth_offset + auth_data_len; 1139 1140 if (auth_dlen > enc_dlen) { 1141 inputlen = auth_dlen + mac_len; 1142 outputlen = auth_dlen; 1143 } else { 1144 inputlen = enc_dlen + mac_len; 1145 outputlen = enc_dlen; 1146 } 1147 1148 if (cpt_ctx->dec_auth != 0) 1149 outputlen = inputlen = enc_dlen; 1150 1151 vq_cmd_w0.s.param1 = encr_data_len; 1152 vq_cmd_w0.s.param2 = auth_data_len; 1153 1154 /* 1155 * In 83XX since we have a limitation of 1156 * IV & Offset control word not part of instruction 1157 * and need to be part of Data Buffer, we check if 1158 * head room is there and then only do the Direct mode processing 1159 */ 1160 if (likely((flags & SINGLE_BUF_INPLACE) && 1161 (flags & SINGLE_BUF_HEADTAILROOM))) { 1162 void *dm_vaddr = fc_params->bufs[0].vaddr; 1163 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1164 /* 1165 * This flag indicates that there is 24 bytes head room and 1166 * 8 bytes tail room available, so that we get to do 1167 * DIRECT MODE with limitation 1168 */ 1169 1170 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1171 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1172 req->ist.ei1 = offset_dma; 1173 1174 /* RPTR should just exclude offset control word */ 1175 req->ist.ei2 = dm_dma_addr - iv_len; 1176 1177 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1178 outputlen - iv_len); 1179 /* since this is decryption, 1180 * don't touch the content of 1181 * alternate ccode space as it contains 1182 * hmac. 1183 */ 1184 1185 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1186 1187 if (likely(iv_len)) { 1188 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1189 OFF_CTRL_LEN); 1190 uint64_t *src = fc_params->iv_buf; 1191 dest[0] = src[0]; 1192 dest[1] = src[1]; 1193 } 1194 1195 if (unlikely((encr_offset >> 16) || 1196 (iv_offset >> 8) || 1197 (auth_offset >> 8))) { 1198 CPT_LOG_DP_ERR("Offset not supported"); 1199 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1200 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1201 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1202 return; 1203 } 1204 1205 *(uint64_t *)offset_vaddr = 1206 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1207 ((uint64_t)iv_offset << 8) | 1208 ((uint64_t)auth_offset)); 1209 1210 } else { 1211 uint64_t dptr_dma, rptr_dma; 1212 uint32_t g_size_bytes, s_size_bytes; 1213 sg_comp_t *gather_comp; 1214 sg_comp_t *scatter_comp; 1215 uint8_t *in_buffer; 1216 uint8_t i = 0; 1217 1218 /* This falls under strict SG mode */ 1219 offset_vaddr = m_vaddr; 1220 offset_dma = m_dma; 1221 size = OFF_CTRL_LEN + iv_len; 1222 1223 m_vaddr = (uint8_t *)m_vaddr + size; 1224 m_dma += size; 1225 1226 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1227 1228 if (likely(iv_len)) { 1229 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1230 OFF_CTRL_LEN); 1231 uint64_t *src = fc_params->iv_buf; 1232 dest[0] = src[0]; 1233 dest[1] = src[1]; 1234 } 1235 1236 if (unlikely((encr_offset >> 16) || 1237 (iv_offset >> 8) || 1238 (auth_offset >> 8))) { 1239 CPT_LOG_DP_ERR("Offset not supported"); 1240 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1241 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1242 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1243 return; 1244 } 1245 1246 *(uint64_t *)offset_vaddr = 1247 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1248 ((uint64_t)iv_offset << 8) | 1249 ((uint64_t)auth_offset)); 1250 1251 /* DPTR has SG list */ 1252 in_buffer = m_vaddr; 1253 dptr_dma = m_dma; 1254 1255 ((uint16_t *)in_buffer)[0] = 0; 1256 ((uint16_t *)in_buffer)[1] = 0; 1257 1258 /* TODO Add error check if space will be sufficient */ 1259 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1260 1261 /* 1262 * Input Gather List 1263 */ 1264 i = 0; 1265 1266 /* Offset control word that includes iv */ 1267 i = fill_sg_comp(gather_comp, i, offset_dma, 1268 OFF_CTRL_LEN + iv_len); 1269 1270 /* Add input data */ 1271 if (flags & VALID_MAC_BUF) { 1272 size = inputlen - iv_len - mac_len; 1273 if (size) { 1274 /* input data only */ 1275 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1276 i = fill_sg_comp_from_buf_min( 1277 gather_comp, i, 1278 fc_params->bufs, 1279 &size); 1280 } else { 1281 uint32_t aad_offset = aad_len ? 1282 passthrough_len : 0; 1283 1284 i = fill_sg_comp_from_iov(gather_comp, 1285 i, 1286 fc_params->src_iov, 1287 0, &size, 1288 aad_buf, 1289 aad_offset); 1290 } 1291 if (unlikely(size)) { 1292 CPT_LOG_DP_ERR("Insufficient buffer" 1293 " space, size %d needed", 1294 size); 1295 return; 1296 } 1297 } 1298 1299 /* mac data */ 1300 if (mac_len) { 1301 i = fill_sg_comp_from_buf(gather_comp, i, 1302 &fc_params->mac_buf); 1303 } 1304 } else { 1305 /* input data + mac */ 1306 size = inputlen - iv_len; 1307 if (size) { 1308 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1309 i = fill_sg_comp_from_buf_min( 1310 gather_comp, i, 1311 fc_params->bufs, 1312 &size); 1313 } else { 1314 uint32_t aad_offset = aad_len ? 1315 passthrough_len : 0; 1316 1317 if (unlikely(!fc_params->src_iov)) { 1318 CPT_LOG_DP_ERR("Bad input args"); 1319 return; 1320 } 1321 1322 i = fill_sg_comp_from_iov( 1323 gather_comp, i, 1324 fc_params->src_iov, 1325 0, &size, 1326 aad_buf, 1327 aad_offset); 1328 } 1329 1330 if (unlikely(size)) { 1331 CPT_LOG_DP_ERR("Insufficient buffer" 1332 " space, size %d needed", 1333 size); 1334 return; 1335 } 1336 } 1337 } 1338 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1339 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1340 1341 /* 1342 * Output Scatter List 1343 */ 1344 1345 i = 0; 1346 scatter_comp = 1347 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1348 1349 /* Add iv */ 1350 if (iv_len) { 1351 i = fill_sg_comp(scatter_comp, i, 1352 offset_dma + OFF_CTRL_LEN, 1353 iv_len); 1354 } 1355 1356 /* Add output data */ 1357 size = outputlen - iv_len; 1358 if (size) { 1359 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1360 /* handle single buffer here */ 1361 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1362 fc_params->bufs, 1363 &size); 1364 } else { 1365 uint32_t aad_offset = aad_len ? 1366 passthrough_len : 0; 1367 1368 if (unlikely(!fc_params->dst_iov)) { 1369 CPT_LOG_DP_ERR("Bad input args"); 1370 return; 1371 } 1372 1373 i = fill_sg_comp_from_iov(scatter_comp, i, 1374 fc_params->dst_iov, 0, 1375 &size, aad_buf, 1376 aad_offset); 1377 } 1378 1379 if (unlikely(size)) { 1380 CPT_LOG_DP_ERR("Insufficient buffer space," 1381 " size %d needed", size); 1382 return; 1383 } 1384 } 1385 1386 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1387 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1388 1389 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1390 1391 /* This is DPTR len incase of SG mode */ 1392 vq_cmd_w0.s.dlen = size; 1393 1394 m_vaddr = (uint8_t *)m_vaddr + size; 1395 m_dma += size; 1396 1397 /* cpt alternate completion address saved earlier */ 1398 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1399 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1400 rptr_dma = c_dma - 8; 1401 size += COMPLETION_CODE_SIZE; 1402 1403 req->ist.ei1 = dptr_dma; 1404 req->ist.ei2 = rptr_dma; 1405 } 1406 1407 /* 16 byte aligned cpt res address */ 1408 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1409 *req->completion_addr = COMPLETION_CODE_INIT; 1410 req->comp_baddr = c_dma; 1411 1412 /* Fill microcode part of instruction */ 1413 req->ist.ei0 = vq_cmd_w0.u64; 1414 1415 req->op = op; 1416 1417 *prep_req = req; 1418 return; 1419 } 1420 1421 static __rte_always_inline void 1422 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1423 uint64_t d_offs, 1424 uint64_t d_lens, 1425 fc_params_t *params, 1426 void *op, 1427 void **prep_req) 1428 { 1429 uint32_t size; 1430 int32_t inputlen, outputlen; 1431 struct cpt_ctx *cpt_ctx; 1432 uint32_t mac_len = 0; 1433 uint8_t snow3g, j; 1434 struct cpt_request_info *req; 1435 buf_ptr_t *buf_p; 1436 uint32_t encr_offset = 0, auth_offset = 0; 1437 uint32_t encr_data_len = 0, auth_data_len = 0; 1438 int flags, iv_len = 16; 1439 void *m_vaddr, *c_vaddr; 1440 uint64_t m_dma, c_dma, offset_ctrl; 1441 uint64_t *offset_vaddr, offset_dma; 1442 uint32_t *iv_s, iv[4]; 1443 vq_cmd_word0_t vq_cmd_w0; 1444 1445 buf_p = ¶ms->meta_buf; 1446 m_vaddr = buf_p->vaddr; 1447 m_dma = buf_p->dma_addr; 1448 1449 cpt_ctx = params->ctx_buf.vaddr; 1450 flags = cpt_ctx->zsk_flags; 1451 mac_len = cpt_ctx->mac_len; 1452 snow3g = cpt_ctx->snow3g; 1453 1454 /* 1455 * Save initial space that followed app data for completion code & 1456 * alternate completion code to fall in same cache line as app data 1457 */ 1458 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1459 m_dma += COMPLETION_CODE_SIZE; 1460 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1461 (uint8_t *)m_vaddr; 1462 1463 c_vaddr = (uint8_t *)m_vaddr + size; 1464 c_dma = m_dma + size; 1465 size += sizeof(cpt_res_s_t); 1466 1467 m_vaddr = (uint8_t *)m_vaddr + size; 1468 m_dma += size; 1469 1470 /* Reserve memory for cpt request info */ 1471 req = m_vaddr; 1472 1473 size = sizeof(struct cpt_request_info); 1474 m_vaddr = (uint8_t *)m_vaddr + size; 1475 m_dma += size; 1476 1477 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1478 1479 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1480 1481 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1482 (0 << 3) | (flags & 0x7)); 1483 1484 if (flags == 0x1) { 1485 /* 1486 * Microcode expects offsets in bytes 1487 * TODO: Rounding off 1488 */ 1489 auth_data_len = AUTH_DLEN(d_lens); 1490 1491 /* EIA3 or UIA2 */ 1492 auth_offset = AUTH_OFFSET(d_offs); 1493 auth_offset = auth_offset / 8; 1494 1495 /* consider iv len */ 1496 auth_offset += iv_len; 1497 1498 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1499 outputlen = mac_len; 1500 1501 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1502 1503 } else { 1504 /* EEA3 or UEA2 */ 1505 /* 1506 * Microcode expects offsets in bytes 1507 * TODO: Rounding off 1508 */ 1509 encr_data_len = ENCR_DLEN(d_lens); 1510 1511 encr_offset = ENCR_OFFSET(d_offs); 1512 encr_offset = encr_offset / 8; 1513 /* consider iv len */ 1514 encr_offset += iv_len; 1515 1516 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1517 outputlen = inputlen; 1518 1519 /* iv offset is 0 */ 1520 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1521 } 1522 1523 if (unlikely((encr_offset >> 16) || 1524 (auth_offset >> 8))) { 1525 CPT_LOG_DP_ERR("Offset not supported"); 1526 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1527 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1528 return; 1529 } 1530 1531 /* IV */ 1532 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1533 params->iv_buf; 1534 1535 if (snow3g) { 1536 /* 1537 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1538 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1539 */ 1540 1541 for (j = 0; j < 4; j++) 1542 iv[j] = iv_s[3 - j]; 1543 } else { 1544 /* ZUC doesn't need a swap */ 1545 for (j = 0; j < 4; j++) 1546 iv[j] = iv_s[j]; 1547 } 1548 1549 /* 1550 * GP op header, lengths are expected in bits. 1551 */ 1552 vq_cmd_w0.s.param1 = encr_data_len; 1553 vq_cmd_w0.s.param2 = auth_data_len; 1554 1555 /* 1556 * In 83XX since we have a limitation of 1557 * IV & Offset control word not part of instruction 1558 * and need to be part of Data Buffer, we check if 1559 * head room is there and then only do the Direct mode processing 1560 */ 1561 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1562 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1563 void *dm_vaddr = params->bufs[0].vaddr; 1564 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1565 /* 1566 * This flag indicates that there is 24 bytes head room and 1567 * 8 bytes tail room available, so that we get to do 1568 * DIRECT MODE with limitation 1569 */ 1570 1571 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1572 OFF_CTRL_LEN - iv_len); 1573 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1574 1575 /* DPTR */ 1576 req->ist.ei1 = offset_dma; 1577 /* RPTR should just exclude offset control word */ 1578 req->ist.ei2 = dm_dma_addr - iv_len; 1579 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1580 + outputlen - iv_len); 1581 1582 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1583 1584 if (likely(iv_len)) { 1585 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1586 + OFF_CTRL_LEN); 1587 memcpy(iv_d, iv, 16); 1588 } 1589 1590 *offset_vaddr = offset_ctrl; 1591 } else { 1592 uint32_t i, g_size_bytes, s_size_bytes; 1593 uint64_t dptr_dma, rptr_dma; 1594 sg_comp_t *gather_comp; 1595 sg_comp_t *scatter_comp; 1596 uint8_t *in_buffer; 1597 uint32_t *iv_d; 1598 1599 /* save space for iv */ 1600 offset_vaddr = m_vaddr; 1601 offset_dma = m_dma; 1602 1603 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1604 m_dma += OFF_CTRL_LEN + iv_len; 1605 1606 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1607 1608 /* DPTR has SG list */ 1609 in_buffer = m_vaddr; 1610 dptr_dma = m_dma; 1611 1612 ((uint16_t *)in_buffer)[0] = 0; 1613 ((uint16_t *)in_buffer)[1] = 0; 1614 1615 /* TODO Add error check if space will be sufficient */ 1616 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1617 1618 /* 1619 * Input Gather List 1620 */ 1621 i = 0; 1622 1623 /* Offset control word followed by iv */ 1624 1625 i = fill_sg_comp(gather_comp, i, offset_dma, 1626 OFF_CTRL_LEN + iv_len); 1627 1628 /* iv offset is 0 */ 1629 *offset_vaddr = offset_ctrl; 1630 1631 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1632 memcpy(iv_d, iv, 16); 1633 1634 /* input data */ 1635 size = inputlen - iv_len; 1636 if (size) { 1637 i = fill_sg_comp_from_iov(gather_comp, i, 1638 params->src_iov, 1639 0, &size, NULL, 0); 1640 if (unlikely(size)) { 1641 CPT_LOG_DP_ERR("Insufficient buffer space," 1642 " size %d needed", size); 1643 return; 1644 } 1645 } 1646 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1647 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1648 1649 /* 1650 * Output Scatter List 1651 */ 1652 1653 i = 0; 1654 scatter_comp = 1655 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1656 1657 if (flags == 0x1) { 1658 /* IV in SLIST only for EEA3 & UEA2 */ 1659 iv_len = 0; 1660 } 1661 1662 if (iv_len) { 1663 i = fill_sg_comp(scatter_comp, i, 1664 offset_dma + OFF_CTRL_LEN, iv_len); 1665 } 1666 1667 /* Add output data */ 1668 if (req_flags & VALID_MAC_BUF) { 1669 size = outputlen - iv_len - mac_len; 1670 if (size) { 1671 i = fill_sg_comp_from_iov(scatter_comp, i, 1672 params->dst_iov, 0, 1673 &size, NULL, 0); 1674 1675 if (unlikely(size)) { 1676 CPT_LOG_DP_ERR("Insufficient buffer space," 1677 " size %d needed", size); 1678 return; 1679 } 1680 } 1681 1682 /* mac data */ 1683 if (mac_len) { 1684 i = fill_sg_comp_from_buf(scatter_comp, i, 1685 ¶ms->mac_buf); 1686 } 1687 } else { 1688 /* Output including mac */ 1689 size = outputlen - iv_len; 1690 if (size) { 1691 i = fill_sg_comp_from_iov(scatter_comp, i, 1692 params->dst_iov, 0, 1693 &size, NULL, 0); 1694 1695 if (unlikely(size)) { 1696 CPT_LOG_DP_ERR("Insufficient buffer space," 1697 " size %d needed", size); 1698 return; 1699 } 1700 } 1701 } 1702 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1703 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1704 1705 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1706 1707 /* This is DPTR len incase of SG mode */ 1708 vq_cmd_w0.s.dlen = size; 1709 1710 m_vaddr = (uint8_t *)m_vaddr + size; 1711 m_dma += size; 1712 1713 /* cpt alternate completion address saved earlier */ 1714 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1715 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1716 rptr_dma = c_dma - 8; 1717 1718 req->ist.ei1 = dptr_dma; 1719 req->ist.ei2 = rptr_dma; 1720 } 1721 1722 /* 16 byte aligned cpt res address */ 1723 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1724 *req->completion_addr = COMPLETION_CODE_INIT; 1725 req->comp_baddr = c_dma; 1726 1727 /* Fill microcode part of instruction */ 1728 req->ist.ei0 = vq_cmd_w0.u64; 1729 1730 req->op = op; 1731 1732 *prep_req = req; 1733 return; 1734 } 1735 1736 static __rte_always_inline void 1737 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1738 uint64_t d_offs, 1739 uint64_t d_lens, 1740 fc_params_t *params, 1741 void *op, 1742 void **prep_req) 1743 { 1744 uint32_t size; 1745 int32_t inputlen = 0, outputlen; 1746 struct cpt_ctx *cpt_ctx; 1747 uint8_t snow3g, iv_len = 16; 1748 struct cpt_request_info *req; 1749 buf_ptr_t *buf_p; 1750 uint32_t encr_offset; 1751 uint32_t encr_data_len; 1752 int flags; 1753 void *m_vaddr, *c_vaddr; 1754 uint64_t m_dma, c_dma; 1755 uint64_t *offset_vaddr, offset_dma; 1756 uint32_t *iv_s, iv[4], j; 1757 vq_cmd_word0_t vq_cmd_w0; 1758 1759 buf_p = ¶ms->meta_buf; 1760 m_vaddr = buf_p->vaddr; 1761 m_dma = buf_p->dma_addr; 1762 1763 /* 1764 * Microcode expects offsets in bytes 1765 * TODO: Rounding off 1766 */ 1767 encr_offset = ENCR_OFFSET(d_offs) / 8; 1768 encr_data_len = ENCR_DLEN(d_lens); 1769 1770 cpt_ctx = params->ctx_buf.vaddr; 1771 flags = cpt_ctx->zsk_flags; 1772 snow3g = cpt_ctx->snow3g; 1773 /* 1774 * Save initial space that followed app data for completion code & 1775 * alternate completion code to fall in same cache line as app data 1776 */ 1777 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1778 m_dma += COMPLETION_CODE_SIZE; 1779 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1780 (uint8_t *)m_vaddr; 1781 1782 c_vaddr = (uint8_t *)m_vaddr + size; 1783 c_dma = m_dma + size; 1784 size += sizeof(cpt_res_s_t); 1785 1786 m_vaddr = (uint8_t *)m_vaddr + size; 1787 m_dma += size; 1788 1789 /* Reserve memory for cpt request info */ 1790 req = m_vaddr; 1791 1792 size = sizeof(struct cpt_request_info); 1793 m_vaddr = (uint8_t *)m_vaddr + size; 1794 m_dma += size; 1795 1796 vq_cmd_w0.u64 = 0; 1797 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1798 1799 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1800 1801 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1802 (0 << 3) | (flags & 0x7)); 1803 1804 /* consider iv len */ 1805 encr_offset += iv_len; 1806 1807 inputlen = encr_offset + 1808 (RTE_ALIGN(encr_data_len, 8) / 8); 1809 outputlen = inputlen; 1810 1811 /* IV */ 1812 iv_s = params->iv_buf; 1813 if (snow3g) { 1814 /* 1815 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1816 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1817 */ 1818 1819 for (j = 0; j < 4; j++) 1820 iv[j] = iv_s[3 - j]; 1821 } else { 1822 /* ZUC doesn't need a swap */ 1823 for (j = 0; j < 4; j++) 1824 iv[j] = iv_s[j]; 1825 } 1826 1827 /* 1828 * GP op header, lengths are expected in bits. 1829 */ 1830 vq_cmd_w0.s.param1 = encr_data_len; 1831 1832 /* 1833 * In 83XX since we have a limitation of 1834 * IV & Offset control word not part of instruction 1835 * and need to be part of Data Buffer, we check if 1836 * head room is there and then only do the Direct mode processing 1837 */ 1838 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1839 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1840 void *dm_vaddr = params->bufs[0].vaddr; 1841 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1842 /* 1843 * This flag indicates that there is 24 bytes head room and 1844 * 8 bytes tail room available, so that we get to do 1845 * DIRECT MODE with limitation 1846 */ 1847 1848 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1849 OFF_CTRL_LEN - iv_len); 1850 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1851 1852 /* DPTR */ 1853 req->ist.ei1 = offset_dma; 1854 /* RPTR should just exclude offset control word */ 1855 req->ist.ei2 = dm_dma_addr - iv_len; 1856 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1857 + outputlen - iv_len); 1858 1859 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1860 1861 if (likely(iv_len)) { 1862 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1863 + OFF_CTRL_LEN); 1864 memcpy(iv_d, iv, 16); 1865 } 1866 1867 /* iv offset is 0 */ 1868 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1869 } else { 1870 uint32_t i, g_size_bytes, s_size_bytes; 1871 uint64_t dptr_dma, rptr_dma; 1872 sg_comp_t *gather_comp; 1873 sg_comp_t *scatter_comp; 1874 uint8_t *in_buffer; 1875 uint32_t *iv_d; 1876 1877 /* save space for offset and iv... */ 1878 offset_vaddr = m_vaddr; 1879 offset_dma = m_dma; 1880 1881 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1882 m_dma += OFF_CTRL_LEN + iv_len; 1883 1884 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1885 1886 /* DPTR has SG list */ 1887 in_buffer = m_vaddr; 1888 dptr_dma = m_dma; 1889 1890 ((uint16_t *)in_buffer)[0] = 0; 1891 ((uint16_t *)in_buffer)[1] = 0; 1892 1893 /* TODO Add error check if space will be sufficient */ 1894 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1895 1896 /* 1897 * Input Gather List 1898 */ 1899 i = 0; 1900 1901 /* Offset control word */ 1902 1903 /* iv offset is 0 */ 1904 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1905 1906 i = fill_sg_comp(gather_comp, i, offset_dma, 1907 OFF_CTRL_LEN + iv_len); 1908 1909 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1910 memcpy(iv_d, iv, 16); 1911 1912 /* Add input data */ 1913 size = inputlen - iv_len; 1914 if (size) { 1915 i = fill_sg_comp_from_iov(gather_comp, i, 1916 params->src_iov, 1917 0, &size, NULL, 0); 1918 if (unlikely(size)) { 1919 CPT_LOG_DP_ERR("Insufficient buffer space," 1920 " size %d needed", size); 1921 return; 1922 } 1923 } 1924 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1925 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1926 1927 /* 1928 * Output Scatter List 1929 */ 1930 1931 i = 0; 1932 scatter_comp = 1933 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1934 1935 /* IV */ 1936 i = fill_sg_comp(scatter_comp, i, 1937 offset_dma + OFF_CTRL_LEN, 1938 iv_len); 1939 1940 /* Add output data */ 1941 size = outputlen - iv_len; 1942 if (size) { 1943 i = fill_sg_comp_from_iov(scatter_comp, i, 1944 params->dst_iov, 0, 1945 &size, NULL, 0); 1946 1947 if (unlikely(size)) { 1948 CPT_LOG_DP_ERR("Insufficient buffer space," 1949 " size %d needed", size); 1950 return; 1951 } 1952 } 1953 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1954 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1955 1956 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1957 1958 /* This is DPTR len incase of SG mode */ 1959 vq_cmd_w0.s.dlen = size; 1960 1961 m_vaddr = (uint8_t *)m_vaddr + size; 1962 m_dma += size; 1963 1964 /* cpt alternate completion address saved earlier */ 1965 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1966 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1967 rptr_dma = c_dma - 8; 1968 1969 req->ist.ei1 = dptr_dma; 1970 req->ist.ei2 = rptr_dma; 1971 } 1972 1973 if (unlikely((encr_offset >> 16))) { 1974 CPT_LOG_DP_ERR("Offset not supported"); 1975 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1976 return; 1977 } 1978 1979 /* 16 byte aligned cpt res address */ 1980 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1981 *req->completion_addr = COMPLETION_CODE_INIT; 1982 req->comp_baddr = c_dma; 1983 1984 /* Fill microcode part of instruction */ 1985 req->ist.ei0 = vq_cmd_w0.u64; 1986 1987 req->op = op; 1988 1989 *prep_req = req; 1990 return; 1991 } 1992 1993 static __rte_always_inline void 1994 cpt_kasumi_enc_prep(uint32_t req_flags, 1995 uint64_t d_offs, 1996 uint64_t d_lens, 1997 fc_params_t *params, 1998 void *op, 1999 void **prep_req) 2000 { 2001 uint32_t size; 2002 int32_t inputlen = 0, outputlen = 0; 2003 struct cpt_ctx *cpt_ctx; 2004 uint32_t mac_len = 0; 2005 uint8_t i = 0; 2006 struct cpt_request_info *req; 2007 buf_ptr_t *buf_p; 2008 uint32_t encr_offset, auth_offset; 2009 uint32_t encr_data_len, auth_data_len; 2010 int flags; 2011 uint8_t *iv_s, *iv_d, iv_len = 8; 2012 uint8_t dir = 0; 2013 void *m_vaddr, *c_vaddr; 2014 uint64_t m_dma, c_dma; 2015 uint64_t *offset_vaddr, offset_dma; 2016 vq_cmd_word0_t vq_cmd_w0; 2017 uint8_t *in_buffer; 2018 uint32_t g_size_bytes, s_size_bytes; 2019 uint64_t dptr_dma, rptr_dma; 2020 sg_comp_t *gather_comp; 2021 sg_comp_t *scatter_comp; 2022 2023 buf_p = ¶ms->meta_buf; 2024 m_vaddr = buf_p->vaddr; 2025 m_dma = buf_p->dma_addr; 2026 2027 encr_offset = ENCR_OFFSET(d_offs) / 8; 2028 auth_offset = AUTH_OFFSET(d_offs) / 8; 2029 encr_data_len = ENCR_DLEN(d_lens); 2030 auth_data_len = AUTH_DLEN(d_lens); 2031 2032 cpt_ctx = params->ctx_buf.vaddr; 2033 flags = cpt_ctx->zsk_flags; 2034 mac_len = cpt_ctx->mac_len; 2035 2036 if (flags == 0x0) 2037 iv_s = params->iv_buf; 2038 else 2039 iv_s = params->auth_iv_buf; 2040 2041 dir = iv_s[8] & 0x1; 2042 2043 /* 2044 * Save initial space that followed app data for completion code & 2045 * alternate completion code to fall in same cache line as app data 2046 */ 2047 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2048 m_dma += COMPLETION_CODE_SIZE; 2049 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2050 (uint8_t *)m_vaddr; 2051 2052 c_vaddr = (uint8_t *)m_vaddr + size; 2053 c_dma = m_dma + size; 2054 size += sizeof(cpt_res_s_t); 2055 2056 m_vaddr = (uint8_t *)m_vaddr + size; 2057 m_dma += size; 2058 2059 /* Reserve memory for cpt request info */ 2060 req = m_vaddr; 2061 2062 size = sizeof(struct cpt_request_info); 2063 m_vaddr = (uint8_t *)m_vaddr + size; 2064 m_dma += size; 2065 2066 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2067 2068 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2069 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2070 (dir << 4) | (0 << 3) | (flags & 0x7)); 2071 2072 /* 2073 * GP op header, lengths are expected in bits. 2074 */ 2075 vq_cmd_w0.s.param1 = encr_data_len; 2076 vq_cmd_w0.s.param2 = auth_data_len; 2077 2078 /* consider iv len */ 2079 if (flags == 0x0) { 2080 encr_offset += iv_len; 2081 auth_offset += iv_len; 2082 } 2083 2084 /* save space for offset ctrl and iv */ 2085 offset_vaddr = m_vaddr; 2086 offset_dma = m_dma; 2087 2088 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2089 m_dma += OFF_CTRL_LEN + iv_len; 2090 2091 /* DPTR has SG list */ 2092 in_buffer = m_vaddr; 2093 dptr_dma = m_dma; 2094 2095 ((uint16_t *)in_buffer)[0] = 0; 2096 ((uint16_t *)in_buffer)[1] = 0; 2097 2098 /* TODO Add error check if space will be sufficient */ 2099 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2100 2101 /* 2102 * Input Gather List 2103 */ 2104 i = 0; 2105 2106 /* Offset control word followed by iv */ 2107 2108 if (flags == 0x0) { 2109 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2110 outputlen = inputlen; 2111 /* iv offset is 0 */ 2112 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2113 if (unlikely((encr_offset >> 16))) { 2114 CPT_LOG_DP_ERR("Offset not supported"); 2115 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 2116 return; 2117 } 2118 } else { 2119 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2120 outputlen = mac_len; 2121 /* iv offset is 0 */ 2122 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2123 if (unlikely((auth_offset >> 8))) { 2124 CPT_LOG_DP_ERR("Offset not supported"); 2125 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 2126 return; 2127 } 2128 } 2129 2130 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2131 2132 /* IV */ 2133 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2134 memcpy(iv_d, iv_s, iv_len); 2135 2136 /* input data */ 2137 size = inputlen - iv_len; 2138 if (size) { 2139 i = fill_sg_comp_from_iov(gather_comp, i, 2140 params->src_iov, 0, 2141 &size, NULL, 0); 2142 2143 if (unlikely(size)) { 2144 CPT_LOG_DP_ERR("Insufficient buffer space," 2145 " size %d needed", size); 2146 return; 2147 } 2148 } 2149 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2150 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2151 2152 /* 2153 * Output Scatter List 2154 */ 2155 2156 i = 0; 2157 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2158 2159 if (flags == 0x1) { 2160 /* IV in SLIST only for F8 */ 2161 iv_len = 0; 2162 } 2163 2164 /* IV */ 2165 if (iv_len) { 2166 i = fill_sg_comp(scatter_comp, i, 2167 offset_dma + OFF_CTRL_LEN, 2168 iv_len); 2169 } 2170 2171 /* Add output data */ 2172 if (req_flags & VALID_MAC_BUF) { 2173 size = outputlen - iv_len - mac_len; 2174 if (size) { 2175 i = fill_sg_comp_from_iov(scatter_comp, i, 2176 params->dst_iov, 0, 2177 &size, NULL, 0); 2178 2179 if (unlikely(size)) { 2180 CPT_LOG_DP_ERR("Insufficient buffer space," 2181 " size %d needed", size); 2182 return; 2183 } 2184 } 2185 2186 /* mac data */ 2187 if (mac_len) { 2188 i = fill_sg_comp_from_buf(scatter_comp, i, 2189 ¶ms->mac_buf); 2190 } 2191 } else { 2192 /* Output including mac */ 2193 size = outputlen - iv_len; 2194 if (size) { 2195 i = fill_sg_comp_from_iov(scatter_comp, i, 2196 params->dst_iov, 0, 2197 &size, NULL, 0); 2198 2199 if (unlikely(size)) { 2200 CPT_LOG_DP_ERR("Insufficient buffer space," 2201 " size %d needed", size); 2202 return; 2203 } 2204 } 2205 } 2206 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2207 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2208 2209 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2210 2211 /* This is DPTR len incase of SG mode */ 2212 vq_cmd_w0.s.dlen = size; 2213 2214 m_vaddr = (uint8_t *)m_vaddr + size; 2215 m_dma += size; 2216 2217 /* cpt alternate completion address saved earlier */ 2218 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2219 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2220 rptr_dma = c_dma - 8; 2221 2222 req->ist.ei1 = dptr_dma; 2223 req->ist.ei2 = rptr_dma; 2224 2225 /* 16 byte aligned cpt res address */ 2226 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2227 *req->completion_addr = COMPLETION_CODE_INIT; 2228 req->comp_baddr = c_dma; 2229 2230 /* Fill microcode part of instruction */ 2231 req->ist.ei0 = vq_cmd_w0.u64; 2232 2233 req->op = op; 2234 2235 *prep_req = req; 2236 return; 2237 } 2238 2239 static __rte_always_inline void 2240 cpt_kasumi_dec_prep(uint64_t d_offs, 2241 uint64_t d_lens, 2242 fc_params_t *params, 2243 void *op, 2244 void **prep_req) 2245 { 2246 uint32_t size; 2247 int32_t inputlen = 0, outputlen; 2248 struct cpt_ctx *cpt_ctx; 2249 uint8_t i = 0, iv_len = 8; 2250 struct cpt_request_info *req; 2251 buf_ptr_t *buf_p; 2252 uint32_t encr_offset; 2253 uint32_t encr_data_len; 2254 int flags; 2255 uint8_t dir = 0; 2256 void *m_vaddr, *c_vaddr; 2257 uint64_t m_dma, c_dma; 2258 uint64_t *offset_vaddr, offset_dma; 2259 vq_cmd_word0_t vq_cmd_w0; 2260 uint8_t *in_buffer; 2261 uint32_t g_size_bytes, s_size_bytes; 2262 uint64_t dptr_dma, rptr_dma; 2263 sg_comp_t *gather_comp; 2264 sg_comp_t *scatter_comp; 2265 2266 buf_p = ¶ms->meta_buf; 2267 m_vaddr = buf_p->vaddr; 2268 m_dma = buf_p->dma_addr; 2269 2270 encr_offset = ENCR_OFFSET(d_offs) / 8; 2271 encr_data_len = ENCR_DLEN(d_lens); 2272 2273 cpt_ctx = params->ctx_buf.vaddr; 2274 flags = cpt_ctx->zsk_flags; 2275 /* 2276 * Save initial space that followed app data for completion code & 2277 * alternate completion code to fall in same cache line as app data 2278 */ 2279 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2280 m_dma += COMPLETION_CODE_SIZE; 2281 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2282 (uint8_t *)m_vaddr; 2283 2284 c_vaddr = (uint8_t *)m_vaddr + size; 2285 c_dma = m_dma + size; 2286 size += sizeof(cpt_res_s_t); 2287 2288 m_vaddr = (uint8_t *)m_vaddr + size; 2289 m_dma += size; 2290 2291 /* Reserve memory for cpt request info */ 2292 req = m_vaddr; 2293 2294 size = sizeof(struct cpt_request_info); 2295 m_vaddr = (uint8_t *)m_vaddr + size; 2296 m_dma += size; 2297 2298 vq_cmd_w0.u64 = 0; 2299 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2300 2301 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2302 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2303 (dir << 4) | (0 << 3) | (flags & 0x7)); 2304 2305 /* 2306 * GP op header, lengths are expected in bits. 2307 */ 2308 vq_cmd_w0.s.param1 = encr_data_len; 2309 2310 /* consider iv len */ 2311 encr_offset += iv_len; 2312 2313 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2314 outputlen = inputlen; 2315 2316 /* save space for offset ctrl & iv */ 2317 offset_vaddr = m_vaddr; 2318 offset_dma = m_dma; 2319 2320 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2321 m_dma += OFF_CTRL_LEN + iv_len; 2322 2323 /* DPTR has SG list */ 2324 in_buffer = m_vaddr; 2325 dptr_dma = m_dma; 2326 2327 ((uint16_t *)in_buffer)[0] = 0; 2328 ((uint16_t *)in_buffer)[1] = 0; 2329 2330 /* TODO Add error check if space will be sufficient */ 2331 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2332 2333 /* 2334 * Input Gather List 2335 */ 2336 i = 0; 2337 2338 /* Offset control word followed by iv */ 2339 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2340 if (unlikely((encr_offset >> 16))) { 2341 CPT_LOG_DP_ERR("Offset not supported"); 2342 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 2343 return; 2344 } 2345 2346 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2347 2348 /* IV */ 2349 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2350 params->iv_buf, iv_len); 2351 2352 /* Add input data */ 2353 size = inputlen - iv_len; 2354 if (size) { 2355 i = fill_sg_comp_from_iov(gather_comp, i, 2356 params->src_iov, 2357 0, &size, NULL, 0); 2358 if (unlikely(size)) { 2359 CPT_LOG_DP_ERR("Insufficient buffer space," 2360 " size %d needed", size); 2361 return; 2362 } 2363 } 2364 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2365 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2366 2367 /* 2368 * Output Scatter List 2369 */ 2370 2371 i = 0; 2372 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2373 2374 /* IV */ 2375 i = fill_sg_comp(scatter_comp, i, 2376 offset_dma + OFF_CTRL_LEN, 2377 iv_len); 2378 2379 /* Add output data */ 2380 size = outputlen - iv_len; 2381 if (size) { 2382 i = fill_sg_comp_from_iov(scatter_comp, i, 2383 params->dst_iov, 0, 2384 &size, NULL, 0); 2385 if (unlikely(size)) { 2386 CPT_LOG_DP_ERR("Insufficient buffer space," 2387 " size %d needed", size); 2388 return; 2389 } 2390 } 2391 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2392 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2393 2394 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2395 2396 /* This is DPTR len incase of SG mode */ 2397 vq_cmd_w0.s.dlen = size; 2398 2399 m_vaddr = (uint8_t *)m_vaddr + size; 2400 m_dma += size; 2401 2402 /* cpt alternate completion address saved earlier */ 2403 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2404 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2405 rptr_dma = c_dma - 8; 2406 2407 req->ist.ei1 = dptr_dma; 2408 req->ist.ei2 = rptr_dma; 2409 2410 /* 16 byte aligned cpt res address */ 2411 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2412 *req->completion_addr = COMPLETION_CODE_INIT; 2413 req->comp_baddr = c_dma; 2414 2415 /* Fill microcode part of instruction */ 2416 req->ist.ei0 = vq_cmd_w0.u64; 2417 2418 req->op = op; 2419 2420 *prep_req = req; 2421 return; 2422 } 2423 2424 static __rte_always_inline void * 2425 cpt_fc_dec_hmac_prep(uint32_t flags, 2426 uint64_t d_offs, 2427 uint64_t d_lens, 2428 fc_params_t *fc_params, 2429 void *op) 2430 { 2431 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2432 uint8_t fc_type; 2433 void *prep_req = NULL; 2434 2435 fc_type = ctx->fc_type; 2436 2437 if (likely(fc_type == FC_GEN)) { 2438 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2439 &prep_req); 2440 } else if (fc_type == ZUC_SNOW3G) { 2441 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2442 &prep_req); 2443 } else if (fc_type == KASUMI) { 2444 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2445 } 2446 2447 /* 2448 * For AUTH_ONLY case, 2449 * MC only supports digest generation and verification 2450 * should be done in software by memcmp() 2451 */ 2452 2453 return prep_req; 2454 } 2455 2456 static __rte_always_inline void *__rte_hot 2457 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2458 fc_params_t *fc_params, void *op) 2459 { 2460 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2461 uint8_t fc_type; 2462 void *prep_req = NULL; 2463 2464 fc_type = ctx->fc_type; 2465 2466 /* Common api for rest of the ops */ 2467 if (likely(fc_type == FC_GEN)) { 2468 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2469 &prep_req); 2470 } else if (fc_type == ZUC_SNOW3G) { 2471 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2472 &prep_req); 2473 } else if (fc_type == KASUMI) { 2474 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2475 &prep_req); 2476 } else if (fc_type == HASH_HMAC) { 2477 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2478 } 2479 2480 return prep_req; 2481 } 2482 2483 static __rte_always_inline int 2484 cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type, 2485 const uint8_t *key, uint16_t key_len, uint16_t mac_len) 2486 { 2487 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 2488 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 2489 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 2490 2491 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2492 uint32_t keyx[4]; 2493 2494 if (key_len != 16) 2495 return -1; 2496 /* No support for AEAD yet */ 2497 if (cpt_ctx->enc_cipher) 2498 return -1; 2499 /* For ZUC/SNOW3G/Kasumi */ 2500 switch (type) { 2501 case SNOW3G_UIA2: 2502 cpt_ctx->snow3g = 1; 2503 gen_key_snow3g(key, keyx); 2504 memcpy(zs_ctx->ci_key, keyx, key_len); 2505 cpt_ctx->fc_type = ZUC_SNOW3G; 2506 cpt_ctx->zsk_flags = 0x1; 2507 break; 2508 case ZUC_EIA3: 2509 cpt_ctx->snow3g = 0; 2510 memcpy(zs_ctx->ci_key, key, key_len); 2511 memcpy(zs_ctx->zuc_const, zuc_d, 32); 2512 cpt_ctx->fc_type = ZUC_SNOW3G; 2513 cpt_ctx->zsk_flags = 0x1; 2514 break; 2515 case KASUMI_F9_ECB: 2516 /* Kasumi ECB mode */ 2517 cpt_ctx->k_ecb = 1; 2518 memcpy(k_ctx->ci_key, key, key_len); 2519 cpt_ctx->fc_type = KASUMI; 2520 cpt_ctx->zsk_flags = 0x1; 2521 break; 2522 case KASUMI_F9_CBC: 2523 memcpy(k_ctx->ci_key, key, key_len); 2524 cpt_ctx->fc_type = KASUMI; 2525 cpt_ctx->zsk_flags = 0x1; 2526 break; 2527 default: 2528 return -1; 2529 } 2530 cpt_ctx->mac_len = 4; 2531 cpt_ctx->hash_type = type; 2532 return 0; 2533 } 2534 2535 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2536 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2537 cpt_ctx->fc_type = HASH_HMAC; 2538 } 2539 2540 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2541 return -1; 2542 2543 /* For GMAC auth, cipher must be NULL */ 2544 if (type == GMAC_TYPE) 2545 fctx->enc.enc_cipher = 0; 2546 2547 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2548 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2549 2550 if (key_len) { 2551 cpt_ctx->hmac = 1; 2552 2553 cpt_ctx->auth_key = rte_zmalloc(NULL, key_len, 8); 2554 if (cpt_ctx->auth_key == NULL) 2555 return -1; 2556 2557 cpt_ctx->auth_key_iova = rte_mem_virt2iova(cpt_ctx->auth_key); 2558 memcpy(cpt_ctx->auth_key, key, key_len); 2559 cpt_ctx->auth_key_len = key_len; 2560 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2561 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2562 2563 if (key_len <= 64) 2564 memcpy(fctx->hmac.opad, key, key_len); 2565 fctx->enc.auth_input_type = 1; 2566 } 2567 return 0; 2568 } 2569 2570 static __rte_always_inline int 2571 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2572 struct cpt_sess_misc *sess) 2573 { 2574 struct rte_crypto_aead_xform *aead_form; 2575 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2576 auth_type_t auth_type = 0; /* NULL Auth type */ 2577 uint32_t cipher_key_len = 0; 2578 uint8_t aes_gcm = 0; 2579 aead_form = &xform->aead; 2580 void *ctx = SESS_PRIV(sess); 2581 2582 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 2583 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2584 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2585 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) { 2586 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2587 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2588 } else { 2589 CPT_LOG_DP_ERR("Unknown aead operation\n"); 2590 return -1; 2591 } 2592 switch (aead_form->algo) { 2593 case RTE_CRYPTO_AEAD_AES_GCM: 2594 enc_type = AES_GCM; 2595 cipher_key_len = 16; 2596 aes_gcm = 1; 2597 break; 2598 case RTE_CRYPTO_AEAD_AES_CCM: 2599 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2600 aead_form->algo); 2601 return -1; 2602 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: 2603 enc_type = CHACHA20; 2604 auth_type = POLY1305; 2605 cipher_key_len = 32; 2606 sess->chacha_poly = 1; 2607 break; 2608 default: 2609 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2610 aead_form->algo); 2611 return -1; 2612 } 2613 if (aead_form->key.length < cipher_key_len) { 2614 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2615 (unsigned int long)aead_form->key.length); 2616 return -1; 2617 } 2618 sess->zsk_flag = 0; 2619 sess->aes_gcm = aes_gcm; 2620 sess->mac_len = aead_form->digest_length; 2621 sess->iv_offset = aead_form->iv.offset; 2622 sess->iv_length = aead_form->iv.length; 2623 sess->aad_length = aead_form->aad_length; 2624 2625 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2626 aead_form->key.length, NULL))) 2627 return -1; 2628 2629 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2630 aead_form->digest_length))) 2631 return -1; 2632 2633 return 0; 2634 } 2635 2636 static __rte_always_inline int 2637 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2638 struct cpt_sess_misc *sess) 2639 { 2640 struct rte_crypto_cipher_xform *c_form; 2641 struct cpt_ctx *ctx = SESS_PRIV(sess); 2642 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2643 uint32_t cipher_key_len = 0; 2644 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2645 2646 c_form = &xform->cipher; 2647 2648 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2649 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2650 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 2651 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2652 if (xform->next != NULL && 2653 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2654 /* Perform decryption followed by auth verify */ 2655 ctx->dec_auth = 1; 2656 } 2657 } else { 2658 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2659 return -1; 2660 } 2661 2662 switch (c_form->algo) { 2663 case RTE_CRYPTO_CIPHER_AES_CBC: 2664 enc_type = AES_CBC; 2665 cipher_key_len = 16; 2666 break; 2667 case RTE_CRYPTO_CIPHER_3DES_CBC: 2668 enc_type = DES3_CBC; 2669 cipher_key_len = 24; 2670 break; 2671 case RTE_CRYPTO_CIPHER_DES_CBC: 2672 /* DES is implemented using 3DES in hardware */ 2673 enc_type = DES3_CBC; 2674 cipher_key_len = 8; 2675 break; 2676 case RTE_CRYPTO_CIPHER_AES_CTR: 2677 enc_type = AES_CTR; 2678 cipher_key_len = 16; 2679 aes_ctr = 1; 2680 break; 2681 case RTE_CRYPTO_CIPHER_NULL: 2682 enc_type = 0; 2683 is_null = 1; 2684 break; 2685 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2686 enc_type = KASUMI_F8_ECB; 2687 cipher_key_len = 16; 2688 zsk_flag = K_F8; 2689 break; 2690 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2691 enc_type = SNOW3G_UEA2; 2692 cipher_key_len = 16; 2693 zsk_flag = ZS_EA; 2694 break; 2695 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2696 enc_type = ZUC_EEA3; 2697 cipher_key_len = 16; 2698 zsk_flag = ZS_EA; 2699 break; 2700 case RTE_CRYPTO_CIPHER_AES_XTS: 2701 enc_type = AES_XTS; 2702 cipher_key_len = 16; 2703 break; 2704 case RTE_CRYPTO_CIPHER_3DES_ECB: 2705 enc_type = DES3_ECB; 2706 cipher_key_len = 24; 2707 break; 2708 case RTE_CRYPTO_CIPHER_AES_ECB: 2709 enc_type = AES_ECB; 2710 cipher_key_len = 16; 2711 break; 2712 case RTE_CRYPTO_CIPHER_3DES_CTR: 2713 case RTE_CRYPTO_CIPHER_AES_F8: 2714 case RTE_CRYPTO_CIPHER_ARC4: 2715 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2716 c_form->algo); 2717 return -1; 2718 default: 2719 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2720 c_form->algo); 2721 return -1; 2722 } 2723 2724 if (c_form->key.length < cipher_key_len) { 2725 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2726 (unsigned long) c_form->key.length); 2727 return -1; 2728 } 2729 2730 sess->zsk_flag = zsk_flag; 2731 sess->aes_gcm = 0; 2732 sess->aes_ctr = aes_ctr; 2733 sess->iv_offset = c_form->iv.offset; 2734 sess->iv_length = c_form->iv.length; 2735 sess->is_null = is_null; 2736 2737 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2738 c_form->key.data, c_form->key.length, NULL))) 2739 return -1; 2740 2741 return 0; 2742 } 2743 2744 static __rte_always_inline int 2745 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2746 struct cpt_sess_misc *sess) 2747 { 2748 struct cpt_ctx *ctx = SESS_PRIV(sess); 2749 struct rte_crypto_auth_xform *a_form; 2750 auth_type_t auth_type = 0; /* NULL Auth type */ 2751 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2752 2753 if (xform->next != NULL && 2754 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2755 xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 2756 /* Perform auth followed by encryption */ 2757 ctx->auth_enc = 1; 2758 } 2759 2760 a_form = &xform->auth; 2761 2762 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2763 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2764 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2765 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2766 else { 2767 CPT_LOG_DP_ERR("Unknown auth operation"); 2768 return -1; 2769 } 2770 2771 switch (a_form->algo) { 2772 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2773 /* Fall through */ 2774 case RTE_CRYPTO_AUTH_SHA1: 2775 auth_type = SHA1_TYPE; 2776 break; 2777 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2778 case RTE_CRYPTO_AUTH_SHA256: 2779 auth_type = SHA2_SHA256; 2780 break; 2781 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2782 case RTE_CRYPTO_AUTH_SHA512: 2783 auth_type = SHA2_SHA512; 2784 break; 2785 case RTE_CRYPTO_AUTH_AES_GMAC: 2786 auth_type = GMAC_TYPE; 2787 aes_gcm = 1; 2788 break; 2789 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2790 case RTE_CRYPTO_AUTH_SHA224: 2791 auth_type = SHA2_SHA224; 2792 break; 2793 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2794 case RTE_CRYPTO_AUTH_SHA384: 2795 auth_type = SHA2_SHA384; 2796 break; 2797 case RTE_CRYPTO_AUTH_MD5_HMAC: 2798 case RTE_CRYPTO_AUTH_MD5: 2799 auth_type = MD5_TYPE; 2800 break; 2801 case RTE_CRYPTO_AUTH_KASUMI_F9: 2802 auth_type = KASUMI_F9_ECB; 2803 /* 2804 * Indicate that direction needs to be taken out 2805 * from end of src 2806 */ 2807 zsk_flag = K_F9; 2808 break; 2809 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2810 auth_type = SNOW3G_UIA2; 2811 zsk_flag = ZS_IA; 2812 break; 2813 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2814 auth_type = ZUC_EIA3; 2815 zsk_flag = ZS_IA; 2816 break; 2817 case RTE_CRYPTO_AUTH_NULL: 2818 auth_type = 0; 2819 is_null = 1; 2820 break; 2821 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2822 case RTE_CRYPTO_AUTH_AES_CMAC: 2823 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2824 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2825 a_form->algo); 2826 return -1; 2827 default: 2828 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2829 a_form->algo); 2830 return -1; 2831 } 2832 2833 sess->zsk_flag = zsk_flag; 2834 sess->aes_gcm = aes_gcm; 2835 sess->mac_len = a_form->digest_length; 2836 sess->is_null = is_null; 2837 if (zsk_flag) { 2838 sess->auth_iv_offset = a_form->iv.offset; 2839 sess->auth_iv_length = a_form->iv.length; 2840 } 2841 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2842 a_form->key.data, a_form->key.length, 2843 a_form->digest_length))) 2844 return -1; 2845 2846 return 0; 2847 } 2848 2849 static __rte_always_inline int 2850 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2851 struct cpt_sess_misc *sess) 2852 { 2853 struct rte_crypto_auth_xform *a_form; 2854 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2855 auth_type_t auth_type = 0; /* NULL Auth type */ 2856 void *ctx = SESS_PRIV(sess); 2857 2858 a_form = &xform->auth; 2859 2860 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2861 sess->cpt_op |= CPT_OP_ENCODE; 2862 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2863 sess->cpt_op |= CPT_OP_DECODE; 2864 else { 2865 CPT_LOG_DP_ERR("Unknown auth operation"); 2866 return -1; 2867 } 2868 2869 switch (a_form->algo) { 2870 case RTE_CRYPTO_AUTH_AES_GMAC: 2871 enc_type = AES_GCM; 2872 auth_type = GMAC_TYPE; 2873 break; 2874 default: 2875 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2876 a_form->algo); 2877 return -1; 2878 } 2879 2880 sess->zsk_flag = 0; 2881 sess->aes_gcm = 0; 2882 sess->is_gmac = 1; 2883 sess->iv_offset = a_form->iv.offset; 2884 sess->iv_length = a_form->iv.length; 2885 sess->mac_len = a_form->digest_length; 2886 2887 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2888 a_form->key.length, NULL))) 2889 return -1; 2890 2891 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2892 a_form->digest_length))) 2893 return -1; 2894 2895 return 0; 2896 } 2897 2898 static __rte_always_inline void * 2899 alloc_op_meta(struct rte_mbuf *m_src, 2900 buf_ptr_t *buf, 2901 int32_t len, 2902 struct rte_mempool *cpt_meta_pool) 2903 { 2904 uint8_t *mdata; 2905 2906 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2907 if (likely(m_src && (m_src->nb_segs == 1))) { 2908 int32_t tailroom; 2909 phys_addr_t mphys; 2910 2911 /* Check if tailroom is sufficient to hold meta data */ 2912 tailroom = rte_pktmbuf_tailroom(m_src); 2913 if (likely(tailroom > len + 8)) { 2914 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2915 mphys = m_src->buf_iova + m_src->buf_len; 2916 mdata -= len; 2917 mphys -= len; 2918 buf->vaddr = mdata; 2919 buf->dma_addr = mphys; 2920 buf->size = len; 2921 /* Indicate that this is a mbuf allocated mdata */ 2922 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2923 return mdata; 2924 } 2925 } 2926 #else 2927 RTE_SET_USED(m_src); 2928 #endif 2929 2930 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2931 return NULL; 2932 2933 buf->vaddr = mdata; 2934 buf->dma_addr = rte_mempool_virt2iova(mdata); 2935 buf->size = len; 2936 2937 return mdata; 2938 } 2939 2940 /** 2941 * cpt_free_metabuf - free metabuf to mempool. 2942 * @param instance: pointer to instance. 2943 * @param objp: pointer to the metabuf. 2944 */ 2945 static __rte_always_inline void 2946 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2947 { 2948 bool nofree = ((uintptr_t)mdata & 1ull); 2949 2950 if (likely(nofree)) 2951 return; 2952 rte_mempool_put(cpt_meta_pool, mdata); 2953 } 2954 2955 static __rte_always_inline uint32_t 2956 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2957 iov_ptr_t *iovec, uint32_t start_offset) 2958 { 2959 uint16_t index = 0; 2960 void *seg_data = NULL; 2961 phys_addr_t seg_phys; 2962 int32_t seg_size = 0; 2963 2964 if (!pkt) { 2965 iovec->buf_cnt = 0; 2966 return 0; 2967 } 2968 2969 if (!start_offset) { 2970 seg_data = rte_pktmbuf_mtod(pkt, void *); 2971 seg_phys = rte_pktmbuf_iova(pkt); 2972 seg_size = pkt->data_len; 2973 } else { 2974 while (start_offset >= pkt->data_len) { 2975 start_offset -= pkt->data_len; 2976 pkt = pkt->next; 2977 } 2978 2979 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2980 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset); 2981 seg_size = pkt->data_len - start_offset; 2982 if (!seg_size) 2983 return 1; 2984 } 2985 2986 /* first seg */ 2987 iovec->bufs[index].vaddr = seg_data; 2988 iovec->bufs[index].dma_addr = seg_phys; 2989 iovec->bufs[index].size = seg_size; 2990 index++; 2991 pkt = pkt->next; 2992 2993 while (unlikely(pkt != NULL)) { 2994 seg_data = rte_pktmbuf_mtod(pkt, void *); 2995 seg_phys = rte_pktmbuf_iova(pkt); 2996 seg_size = pkt->data_len; 2997 if (!seg_size) 2998 break; 2999 3000 iovec->bufs[index].vaddr = seg_data; 3001 iovec->bufs[index].dma_addr = seg_phys; 3002 iovec->bufs[index].size = seg_size; 3003 3004 index++; 3005 3006 pkt = pkt->next; 3007 } 3008 3009 iovec->buf_cnt = index; 3010 return 0; 3011 } 3012 3013 static __rte_always_inline uint32_t 3014 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 3015 fc_params_t *param, 3016 uint32_t *flags) 3017 { 3018 uint16_t index = 0; 3019 void *seg_data = NULL; 3020 phys_addr_t seg_phys; 3021 uint32_t seg_size = 0; 3022 iov_ptr_t *iovec; 3023 3024 seg_data = rte_pktmbuf_mtod(pkt, void *); 3025 seg_phys = rte_pktmbuf_iova(pkt); 3026 seg_size = pkt->data_len; 3027 3028 /* first seg */ 3029 if (likely(!pkt->next)) { 3030 uint32_t headroom, tailroom; 3031 3032 *flags |= SINGLE_BUF_INPLACE; 3033 headroom = rte_pktmbuf_headroom(pkt); 3034 tailroom = rte_pktmbuf_tailroom(pkt); 3035 if (likely((headroom >= 24) && 3036 (tailroom >= 8))) { 3037 /* In 83XX this is prerequisite for Direct mode */ 3038 *flags |= SINGLE_BUF_HEADTAILROOM; 3039 } 3040 param->bufs[0].vaddr = seg_data; 3041 param->bufs[0].dma_addr = seg_phys; 3042 param->bufs[0].size = seg_size; 3043 return 0; 3044 } 3045 iovec = param->src_iov; 3046 iovec->bufs[index].vaddr = seg_data; 3047 iovec->bufs[index].dma_addr = seg_phys; 3048 iovec->bufs[index].size = seg_size; 3049 index++; 3050 pkt = pkt->next; 3051 3052 while (unlikely(pkt != NULL)) { 3053 seg_data = rte_pktmbuf_mtod(pkt, void *); 3054 seg_phys = rte_pktmbuf_iova(pkt); 3055 seg_size = pkt->data_len; 3056 3057 if (!seg_size) 3058 break; 3059 3060 iovec->bufs[index].vaddr = seg_data; 3061 iovec->bufs[index].dma_addr = seg_phys; 3062 iovec->bufs[index].size = seg_size; 3063 3064 index++; 3065 3066 pkt = pkt->next; 3067 } 3068 3069 iovec->buf_cnt = index; 3070 return 0; 3071 } 3072 3073 static __rte_always_inline int 3074 fill_fc_params(struct rte_crypto_op *cop, 3075 struct cpt_sess_misc *sess_misc, 3076 struct cpt_qp_meta_info *m_info, 3077 void **mdata_ptr, 3078 void **prep_req) 3079 { 3080 uint32_t space = 0; 3081 struct rte_crypto_sym_op *sym_op = cop->sym; 3082 struct cpt_ctx *ctx = SESS_PRIV(sess_misc); 3083 void *mdata = NULL; 3084 uintptr_t *op; 3085 uint32_t mc_hash_off; 3086 uint32_t flags = 0; 3087 uint64_t d_offs, d_lens; 3088 struct rte_mbuf *m_src, *m_dst; 3089 uint8_t cpt_op = sess_misc->cpt_op; 3090 #ifdef CPT_ALWAYS_USE_SG_MODE 3091 uint8_t inplace = 0; 3092 #else 3093 uint8_t inplace = 1; 3094 #endif 3095 fc_params_t fc_params; 3096 char src[SRC_IOV_SIZE]; 3097 char dst[SRC_IOV_SIZE]; 3098 uint32_t iv_buf[4]; 3099 int ret; 3100 3101 if (likely(sess_misc->iv_length)) { 3102 flags |= VALID_IV_BUF; 3103 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3104 uint8_t *, sess_misc->iv_offset); 3105 if (sess_misc->aes_ctr && 3106 unlikely(sess_misc->iv_length != 16)) { 3107 memcpy((uint8_t *)iv_buf, 3108 rte_crypto_op_ctod_offset(cop, 3109 uint8_t *, sess_misc->iv_offset), 12); 3110 iv_buf[3] = rte_cpu_to_be_32(0x1); 3111 fc_params.iv_buf = iv_buf; 3112 } 3113 } 3114 3115 if (sess_misc->zsk_flag) { 3116 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3117 uint8_t *, 3118 sess_misc->auth_iv_offset); 3119 if (sess_misc->zsk_flag != ZS_EA) 3120 inplace = 0; 3121 } 3122 m_src = sym_op->m_src; 3123 m_dst = sym_op->m_dst; 3124 3125 if (sess_misc->aes_gcm || sess_misc->chacha_poly) { 3126 uint8_t *salt; 3127 uint8_t *aad_data; 3128 uint16_t aad_len; 3129 3130 d_offs = sym_op->aead.data.offset; 3131 d_lens = sym_op->aead.data.length; 3132 mc_hash_off = sym_op->aead.data.offset + 3133 sym_op->aead.data.length; 3134 3135 aad_data = sym_op->aead.aad.data; 3136 aad_len = sess_misc->aad_length; 3137 if (likely((aad_data + aad_len) == 3138 rte_pktmbuf_mtod_offset(m_src, 3139 uint8_t *, 3140 sym_op->aead.data.offset))) { 3141 d_offs = (d_offs - aad_len) | (d_offs << 16); 3142 d_lens = (d_lens + aad_len) | (d_lens << 32); 3143 } else { 3144 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3145 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3146 fc_params.aad_buf.size = aad_len; 3147 flags |= VALID_AAD_BUF; 3148 inplace = 0; 3149 d_offs = d_offs << 16; 3150 d_lens = d_lens << 32; 3151 } 3152 3153 salt = fc_params.iv_buf; 3154 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3155 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3156 sess_misc->salt = *(uint32_t *)salt; 3157 } 3158 fc_params.iv_buf = salt + 4; 3159 if (likely(sess_misc->mac_len)) { 3160 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3161 m_src; 3162 3163 if (!m) 3164 m = m_src; 3165 3166 /* hmac immediately following data is best case */ 3167 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3168 mc_hash_off != 3169 (uint8_t *)sym_op->aead.digest.data)) { 3170 flags |= VALID_MAC_BUF; 3171 fc_params.mac_buf.size = sess_misc->mac_len; 3172 fc_params.mac_buf.vaddr = 3173 sym_op->aead.digest.data; 3174 fc_params.mac_buf.dma_addr = 3175 sym_op->aead.digest.phys_addr; 3176 inplace = 0; 3177 } 3178 } 3179 } else { 3180 d_offs = sym_op->cipher.data.offset; 3181 d_lens = sym_op->cipher.data.length; 3182 mc_hash_off = sym_op->cipher.data.offset + 3183 sym_op->cipher.data.length; 3184 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3185 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3186 3187 if (mc_hash_off < (sym_op->auth.data.offset + 3188 sym_op->auth.data.length)){ 3189 mc_hash_off = (sym_op->auth.data.offset + 3190 sym_op->auth.data.length); 3191 } 3192 /* for gmac, salt should be updated like in gcm */ 3193 if (unlikely(sess_misc->is_gmac)) { 3194 uint8_t *salt; 3195 salt = fc_params.iv_buf; 3196 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3197 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3198 sess_misc->salt = *(uint32_t *)salt; 3199 } 3200 fc_params.iv_buf = salt + 4; 3201 } 3202 if (likely(sess_misc->mac_len)) { 3203 struct rte_mbuf *m; 3204 3205 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3206 if (!m) 3207 m = m_src; 3208 3209 /* hmac immediately following data is best case */ 3210 if (!ctx->dec_auth && !ctx->auth_enc && 3211 (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3212 mc_hash_off != 3213 (uint8_t *)sym_op->auth.digest.data))) { 3214 flags |= VALID_MAC_BUF; 3215 fc_params.mac_buf.size = 3216 sess_misc->mac_len; 3217 fc_params.mac_buf.vaddr = 3218 sym_op->auth.digest.data; 3219 fc_params.mac_buf.dma_addr = 3220 sym_op->auth.digest.phys_addr; 3221 inplace = 0; 3222 } 3223 } 3224 } 3225 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3226 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3227 3228 if (!ctx->dec_auth && 3229 unlikely(sess_misc->is_null || 3230 sess_misc->cpt_op == CPT_OP_DECODE)) 3231 inplace = 0; 3232 3233 if (likely(!m_dst && inplace)) { 3234 /* Case of single buffer without AAD buf or 3235 * separate mac buf in place and 3236 * not air crypto 3237 */ 3238 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3239 3240 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3241 &fc_params, 3242 &flags))) { 3243 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3244 ret = -EINVAL; 3245 goto err_exit; 3246 } 3247 3248 } else { 3249 /* Out of place processing */ 3250 fc_params.src_iov = (void *)src; 3251 fc_params.dst_iov = (void *)dst; 3252 3253 /* Store SG I/O in the api for reuse */ 3254 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3255 CPT_LOG_DP_ERR("Prepare src iov failed"); 3256 ret = -EINVAL; 3257 goto err_exit; 3258 } 3259 3260 if (unlikely(m_dst != NULL)) { 3261 uint32_t pkt_len; 3262 3263 /* Try to make room as much as src has */ 3264 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3265 3266 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3267 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3268 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3269 CPT_LOG_DP_ERR("Not enough space in " 3270 "m_dst %p, need %u" 3271 " more", 3272 m_dst, pkt_len); 3273 ret = -EINVAL; 3274 goto err_exit; 3275 } 3276 } 3277 3278 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3279 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3280 "m_dst %p", m_dst); 3281 ret = -EINVAL; 3282 goto err_exit; 3283 } 3284 } else { 3285 fc_params.dst_iov = (void *)src; 3286 } 3287 } 3288 3289 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3290 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3291 m_info->lb_mlen, m_info->pool); 3292 else 3293 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3294 m_info->sg_mlen, m_info->pool); 3295 3296 if (unlikely(mdata == NULL)) { 3297 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3298 ret = -ENOMEM; 3299 goto err_exit; 3300 } 3301 3302 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3303 op[0] = (uintptr_t)mdata; 3304 op[1] = (uintptr_t)cop; 3305 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3306 space += 4 * sizeof(uint64_t); 3307 3308 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3309 fc_params.meta_buf.dma_addr += space; 3310 fc_params.meta_buf.size -= space; 3311 3312 /* Finally prepare the instruction */ 3313 if (cpt_op & CPT_OP_ENCODE) 3314 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3315 &fc_params, op); 3316 else 3317 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3318 &fc_params, op); 3319 3320 if (unlikely(*prep_req == NULL)) { 3321 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3322 ret = -EINVAL; 3323 goto free_mdata_and_exit; 3324 } 3325 3326 *mdata_ptr = mdata; 3327 3328 return 0; 3329 3330 free_mdata_and_exit: 3331 free_op_meta(mdata, m_info->pool); 3332 err_exit: 3333 return ret; 3334 } 3335 3336 static __rte_always_inline void 3337 compl_auth_verify(struct rte_crypto_op *op, 3338 uint8_t *gen_mac, 3339 uint64_t mac_len) 3340 { 3341 uint8_t *mac; 3342 struct rte_crypto_sym_op *sym_op = op->sym; 3343 3344 if (sym_op->auth.digest.data) 3345 mac = sym_op->auth.digest.data; 3346 else 3347 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3348 uint8_t *, 3349 sym_op->auth.data.length + 3350 sym_op->auth.data.offset); 3351 if (!mac) { 3352 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3353 return; 3354 } 3355 3356 if (memcmp(mac, gen_mac, mac_len)) 3357 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3358 else 3359 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3360 } 3361 3362 static __rte_always_inline void 3363 find_kasumif9_direction_and_length(uint8_t *src, 3364 uint32_t counter_num_bytes, 3365 uint32_t *addr_length_in_bits, 3366 uint8_t *addr_direction) 3367 { 3368 uint8_t found = 0; 3369 uint32_t pos; 3370 uint8_t last_byte; 3371 while (!found && counter_num_bytes > 0) { 3372 counter_num_bytes--; 3373 if (src[counter_num_bytes] == 0x00) 3374 continue; 3375 pos = rte_bsf32(src[counter_num_bytes]); 3376 if (pos == 7) { 3377 if (likely(counter_num_bytes > 0)) { 3378 last_byte = src[counter_num_bytes - 1]; 3379 *addr_direction = last_byte & 0x1; 3380 *addr_length_in_bits = counter_num_bytes * 8 3381 - 1; 3382 } 3383 } else { 3384 last_byte = src[counter_num_bytes]; 3385 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3386 *addr_length_in_bits = counter_num_bytes * 8 3387 + (8 - (pos + 2)); 3388 } 3389 found = 1; 3390 } 3391 } 3392 3393 /* 3394 * This handles all auth only except AES_GMAC 3395 */ 3396 static __rte_always_inline int 3397 fill_digest_params(struct rte_crypto_op *cop, 3398 struct cpt_sess_misc *sess, 3399 struct cpt_qp_meta_info *m_info, 3400 void **mdata_ptr, 3401 void **prep_req) 3402 { 3403 uint32_t space = 0; 3404 struct rte_crypto_sym_op *sym_op = cop->sym; 3405 void *mdata; 3406 phys_addr_t mphys; 3407 uint64_t *op; 3408 uint32_t auth_range_off; 3409 uint32_t flags = 0; 3410 uint64_t d_offs = 0, d_lens; 3411 struct rte_mbuf *m_src, *m_dst; 3412 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3413 uint16_t mac_len = sess->mac_len; 3414 fc_params_t params; 3415 char src[SRC_IOV_SIZE]; 3416 uint8_t iv_buf[16]; 3417 int ret; 3418 3419 memset(¶ms, 0, sizeof(fc_params_t)); 3420 3421 m_src = sym_op->m_src; 3422 3423 /* For just digest lets force mempool alloc */ 3424 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3425 m_info->pool); 3426 if (mdata == NULL) { 3427 ret = -ENOMEM; 3428 goto err_exit; 3429 } 3430 3431 mphys = params.meta_buf.dma_addr; 3432 3433 op = mdata; 3434 op[0] = (uintptr_t)mdata; 3435 op[1] = (uintptr_t)cop; 3436 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3437 space += 4 * sizeof(uint64_t); 3438 3439 auth_range_off = sym_op->auth.data.offset; 3440 3441 flags = VALID_MAC_BUF; 3442 params.src_iov = (void *)src; 3443 if (unlikely(sess->zsk_flag)) { 3444 /* 3445 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3446 * we will send pass through even for auth only case, 3447 * let MC handle it 3448 */ 3449 d_offs = auth_range_off; 3450 auth_range_off = 0; 3451 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3452 uint8_t *, sess->auth_iv_offset); 3453 if (sess->zsk_flag == K_F9) { 3454 uint32_t length_in_bits, num_bytes; 3455 uint8_t *src, direction = 0; 3456 3457 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3458 uint8_t *), 8); 3459 /* 3460 * This is kasumi f9, take direction from 3461 * source buffer 3462 */ 3463 length_in_bits = cop->sym->auth.data.length; 3464 num_bytes = (length_in_bits >> 3); 3465 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3466 find_kasumif9_direction_and_length(src, 3467 num_bytes, 3468 &length_in_bits, 3469 &direction); 3470 length_in_bits -= 64; 3471 cop->sym->auth.data.offset += 64; 3472 d_offs = cop->sym->auth.data.offset; 3473 auth_range_off = d_offs / 8; 3474 cop->sym->auth.data.length = length_in_bits; 3475 3476 /* Store it at end of auth iv */ 3477 iv_buf[8] = direction; 3478 params.auth_iv_buf = iv_buf; 3479 } 3480 } 3481 3482 d_lens = sym_op->auth.data.length; 3483 3484 params.ctx_buf.vaddr = SESS_PRIV(sess); 3485 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3486 3487 if (auth_op == CPT_OP_AUTH_GENERATE) { 3488 if (sym_op->auth.digest.data) { 3489 /* 3490 * Digest to be generated 3491 * in separate buffer 3492 */ 3493 params.mac_buf.size = 3494 sess->mac_len; 3495 params.mac_buf.vaddr = 3496 sym_op->auth.digest.data; 3497 params.mac_buf.dma_addr = 3498 sym_op->auth.digest.phys_addr; 3499 } else { 3500 uint32_t off = sym_op->auth.data.offset + 3501 sym_op->auth.data.length; 3502 int32_t dlen, space; 3503 3504 m_dst = sym_op->m_dst ? 3505 sym_op->m_dst : sym_op->m_src; 3506 dlen = rte_pktmbuf_pkt_len(m_dst); 3507 3508 space = off + mac_len - dlen; 3509 if (space > 0) 3510 if (!rte_pktmbuf_append(m_dst, space)) { 3511 CPT_LOG_DP_ERR("Failed to extend " 3512 "mbuf by %uB", space); 3513 ret = -EINVAL; 3514 goto free_mdata_and_exit; 3515 } 3516 3517 params.mac_buf.vaddr = 3518 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3519 params.mac_buf.dma_addr = 3520 rte_pktmbuf_iova_offset(m_dst, off); 3521 params.mac_buf.size = mac_len; 3522 } 3523 } else { 3524 /* Need space for storing generated mac */ 3525 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3526 params.mac_buf.dma_addr = mphys + space; 3527 params.mac_buf.size = mac_len; 3528 space += RTE_ALIGN_CEIL(mac_len, 8); 3529 op[2] = (uintptr_t)params.mac_buf.vaddr; 3530 op[3] = mac_len; 3531 } 3532 3533 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3534 params.meta_buf.dma_addr = mphys + space; 3535 params.meta_buf.size -= space; 3536 3537 /* Out of place processing */ 3538 params.src_iov = (void *)src; 3539 3540 /*Store SG I/O in the api for reuse */ 3541 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3542 CPT_LOG_DP_ERR("Prepare src iov failed"); 3543 ret = -EINVAL; 3544 goto free_mdata_and_exit; 3545 } 3546 3547 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3548 if (unlikely(*prep_req == NULL)) { 3549 ret = -EINVAL; 3550 goto free_mdata_and_exit; 3551 } 3552 3553 *mdata_ptr = mdata; 3554 3555 return 0; 3556 3557 free_mdata_and_exit: 3558 free_op_meta(mdata, m_info->pool); 3559 err_exit: 3560 return ret; 3561 } 3562 3563 #endif /*_CPT_UCODE_H_ */ 3564