1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline void 26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 27 { 28 int i, base; 29 30 for (i = 0; i < 4; i++) { 31 base = 4 * i; 32 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 33 (ck[base + 2] << 8) | (ck[base + 3]); 34 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 35 } 36 } 37 38 static __rte_always_inline int 39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) 40 { 41 uint16_t mac_len = auth->digest_length; 42 int ret; 43 44 switch (auth->algo) { 45 case RTE_CRYPTO_AUTH_MD5: 46 case RTE_CRYPTO_AUTH_MD5_HMAC: 47 ret = (mac_len == 16) ? 0 : -1; 48 break; 49 case RTE_CRYPTO_AUTH_SHA1: 50 case RTE_CRYPTO_AUTH_SHA1_HMAC: 51 ret = (mac_len == 20) ? 0 : -1; 52 break; 53 case RTE_CRYPTO_AUTH_SHA224: 54 case RTE_CRYPTO_AUTH_SHA224_HMAC: 55 ret = (mac_len == 28) ? 0 : -1; 56 break; 57 case RTE_CRYPTO_AUTH_SHA256: 58 case RTE_CRYPTO_AUTH_SHA256_HMAC: 59 ret = (mac_len == 32) ? 0 : -1; 60 break; 61 case RTE_CRYPTO_AUTH_SHA384: 62 case RTE_CRYPTO_AUTH_SHA384_HMAC: 63 ret = (mac_len == 48) ? 0 : -1; 64 break; 65 case RTE_CRYPTO_AUTH_SHA512: 66 case RTE_CRYPTO_AUTH_SHA512_HMAC: 67 ret = (mac_len == 64) ? 0 : -1; 68 break; 69 case RTE_CRYPTO_AUTH_NULL: 70 ret = 0; 71 break; 72 default: 73 ret = -1; 74 } 75 76 return ret; 77 } 78 79 static __rte_always_inline void 80 cpt_fc_salt_update(struct cpt_ctx *cpt_ctx, 81 uint8_t *salt) 82 { 83 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 84 memcpy(fctx->enc.encr_iv, salt, 4); 85 } 86 87 static __rte_always_inline int 88 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 89 { 90 switch (key_len) { 91 case 16: 92 case 24: 93 case 32: 94 return 0; 95 default: 96 return -1; 97 } 98 } 99 100 static __rte_always_inline int 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 102 { 103 int fc_type = 0; 104 switch (type) { 105 case PASSTHROUGH: 106 fc_type = FC_GEN; 107 break; 108 case DES3_CBC: 109 case DES3_ECB: 110 fc_type = FC_GEN; 111 break; 112 case AES_CBC: 113 case AES_ECB: 114 case AES_CFB: 115 case AES_CTR: 116 case AES_GCM: 117 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 118 return -1; 119 fc_type = FC_GEN; 120 break; 121 case CHACHA20: 122 fc_type = FC_GEN; 123 break; 124 case AES_XTS: 125 key_len = key_len / 2; 126 if (unlikely(key_len == 24)) { 127 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 128 return -1; 129 } 130 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 131 return -1; 132 fc_type = FC_GEN; 133 break; 134 case ZUC_EEA3: 135 case SNOW3G_UEA2: 136 if (unlikely(key_len != 16)) 137 return -1; 138 /* No support for AEAD yet */ 139 if (unlikely(ctx->hash_type)) 140 return -1; 141 fc_type = ZUC_SNOW3G; 142 break; 143 case KASUMI_F8_CBC: 144 case KASUMI_F8_ECB: 145 if (unlikely(key_len != 16)) 146 return -1; 147 /* No support for AEAD yet */ 148 if (unlikely(ctx->hash_type)) 149 return -1; 150 fc_type = KASUMI; 151 break; 152 default: 153 return -1; 154 } 155 156 ctx->fc_type = fc_type; 157 return 0; 158 } 159 160 static __rte_always_inline void 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 162 { 163 cpt_ctx->enc_cipher = 0; 164 fctx->enc.enc_cipher = 0; 165 } 166 167 static __rte_always_inline void 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 169 { 170 mc_aes_type_t aes_key_type = 0; 171 switch (key_len) { 172 case 16: 173 aes_key_type = AES_128_BIT; 174 break; 175 case 24: 176 aes_key_type = AES_192_BIT; 177 break; 178 case 32: 179 aes_key_type = AES_256_BIT; 180 break; 181 default: 182 /* This should not happen */ 183 CPT_LOG_DP_ERR("Invalid AES key len"); 184 return; 185 } 186 fctx->enc.aes_key = aes_key_type; 187 } 188 189 static __rte_always_inline void 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 191 uint16_t key_len) 192 { 193 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 194 uint32_t keyx[4]; 195 196 cpt_ctx->snow3g = 1; 197 gen_key_snow3g(key, keyx); 198 memcpy(zs_ctx->ci_key, keyx, key_len); 199 cpt_ctx->zsk_flags = 0; 200 } 201 202 static __rte_always_inline void 203 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 204 uint16_t key_len) 205 { 206 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 207 208 cpt_ctx->snow3g = 0; 209 memcpy(zs_ctx->ci_key, key, key_len); 210 memcpy(zs_ctx->zuc_const, zuc_d, 32); 211 cpt_ctx->zsk_flags = 0; 212 } 213 214 static __rte_always_inline void 215 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 216 uint16_t key_len) 217 { 218 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 219 220 cpt_ctx->k_ecb = 1; 221 memcpy(k_ctx->ci_key, key, key_len); 222 cpt_ctx->zsk_flags = 0; 223 } 224 225 static __rte_always_inline void 226 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 227 uint16_t key_len) 228 { 229 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 230 231 memcpy(k_ctx->ci_key, key, key_len); 232 cpt_ctx->zsk_flags = 0; 233 } 234 235 static __rte_always_inline int 236 cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type, 237 const uint8_t *key, uint16_t key_len, uint8_t *salt) 238 { 239 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 240 int ret; 241 242 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 243 if (unlikely(ret)) 244 return -1; 245 246 if (cpt_ctx->fc_type == FC_GEN) { 247 /* 248 * We need to always say IV is from DPTR as user can 249 * sometimes iverride IV per operation. 250 */ 251 fctx->enc.iv_source = CPT_FROM_DPTR; 252 253 if (cpt_ctx->auth_key_len > 64) 254 return -1; 255 } 256 257 switch (type) { 258 case PASSTHROUGH: 259 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 260 goto success; 261 case DES3_CBC: 262 /* CPT performs DES using 3DES with the 8B DES-key 263 * replicated 2 more times to match the 24B 3DES-key. 264 * Eg. If org. key is "0x0a 0x0b", then new key is 265 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 266 */ 267 if (key_len == 8) { 268 /* Skipping the first 8B as it will be copied 269 * in the regular code flow 270 */ 271 memcpy(fctx->enc.encr_key+key_len, key, key_len); 272 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 273 } 274 break; 275 case DES3_ECB: 276 /* For DES3_ECB IV need to be from CTX. */ 277 fctx->enc.iv_source = CPT_FROM_CTX; 278 break; 279 case AES_CBC: 280 case AES_ECB: 281 case AES_CFB: 282 case AES_CTR: 283 case CHACHA20: 284 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 285 break; 286 case AES_GCM: 287 /* Even though iv source is from dptr, 288 * aes_gcm salt is taken from ctx 289 */ 290 if (salt) { 291 memcpy(fctx->enc.encr_iv, salt, 4); 292 /* Assuming it was just salt update 293 * and nothing else 294 */ 295 if (!key) 296 goto success; 297 } 298 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 299 break; 300 case AES_XTS: 301 key_len = key_len / 2; 302 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 303 304 /* Copy key2 for XTS into ipad */ 305 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 306 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 307 break; 308 case SNOW3G_UEA2: 309 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 310 goto success; 311 case ZUC_EEA3: 312 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 313 goto success; 314 case KASUMI_F8_ECB: 315 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 316 goto success; 317 case KASUMI_F8_CBC: 318 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 319 goto success; 320 default: 321 return -1; 322 } 323 324 /* Only for FC_GEN case */ 325 326 /* For GMAC auth, cipher must be NULL */ 327 if (cpt_ctx->hash_type != GMAC_TYPE) 328 fctx->enc.enc_cipher = type; 329 330 memcpy(fctx->enc.encr_key, key, key_len); 331 332 success: 333 cpt_ctx->enc_cipher = type; 334 335 return 0; 336 } 337 338 static __rte_always_inline uint32_t 339 fill_sg_comp(sg_comp_t *list, 340 uint32_t i, 341 phys_addr_t dma_addr, 342 uint32_t size) 343 { 344 sg_comp_t *to = &list[i>>2]; 345 346 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 347 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 348 i++; 349 return i; 350 } 351 352 static __rte_always_inline uint32_t 353 fill_sg_comp_from_buf(sg_comp_t *list, 354 uint32_t i, 355 buf_ptr_t *from) 356 { 357 sg_comp_t *to = &list[i>>2]; 358 359 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 360 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 361 i++; 362 return i; 363 } 364 365 static __rte_always_inline uint32_t 366 fill_sg_comp_from_buf_min(sg_comp_t *list, 367 uint32_t i, 368 buf_ptr_t *from, 369 uint32_t *psize) 370 { 371 sg_comp_t *to = &list[i >> 2]; 372 uint32_t size = *psize; 373 uint32_t e_len; 374 375 e_len = (size > from->size) ? from->size : size; 376 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 377 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 378 *psize -= e_len; 379 i++; 380 return i; 381 } 382 383 /* 384 * This fills the MC expected SGIO list 385 * from IOV given by user. 386 */ 387 static __rte_always_inline uint32_t 388 fill_sg_comp_from_iov(sg_comp_t *list, 389 uint32_t i, 390 iov_ptr_t *from, uint32_t from_offset, 391 uint32_t *psize, buf_ptr_t *extra_buf, 392 uint32_t extra_offset) 393 { 394 int32_t j; 395 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 396 uint32_t size = *psize; 397 buf_ptr_t *bufs; 398 399 bufs = from->bufs; 400 for (j = 0; (j < from->buf_cnt) && size; j++) { 401 phys_addr_t e_dma_addr; 402 uint32_t e_len; 403 sg_comp_t *to = &list[i >> 2]; 404 405 if (unlikely(from_offset)) { 406 if (from_offset >= bufs[j].size) { 407 from_offset -= bufs[j].size; 408 continue; 409 } 410 e_dma_addr = bufs[j].dma_addr + from_offset; 411 e_len = (size > (bufs[j].size - from_offset)) ? 412 (bufs[j].size - from_offset) : size; 413 from_offset = 0; 414 } else { 415 e_dma_addr = bufs[j].dma_addr; 416 e_len = (size > bufs[j].size) ? 417 bufs[j].size : size; 418 } 419 420 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 421 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 422 423 if (extra_len && (e_len >= extra_offset)) { 424 /* Break the data at given offset */ 425 uint32_t next_len = e_len - extra_offset; 426 phys_addr_t next_dma = e_dma_addr + extra_offset; 427 428 if (!extra_offset) { 429 i--; 430 } else { 431 e_len = extra_offset; 432 size -= e_len; 433 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 434 } 435 436 extra_len = RTE_MIN(extra_len, size); 437 /* Insert extra data ptr */ 438 if (extra_len) { 439 i++; 440 to = &list[i >> 2]; 441 to->u.s.len[i % 4] = 442 rte_cpu_to_be_16(extra_len); 443 to->ptr[i % 4] = 444 rte_cpu_to_be_64(extra_buf->dma_addr); 445 size -= extra_len; 446 } 447 448 next_len = RTE_MIN(next_len, size); 449 /* insert the rest of the data */ 450 if (next_len) { 451 i++; 452 to = &list[i >> 2]; 453 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 454 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 455 size -= next_len; 456 } 457 extra_len = 0; 458 459 } else { 460 size -= e_len; 461 } 462 if (extra_offset) 463 extra_offset -= size; 464 i++; 465 } 466 467 *psize = size; 468 return (uint32_t)i; 469 } 470 471 static __rte_always_inline void 472 cpt_digest_gen_prep(uint32_t flags, 473 uint64_t d_lens, 474 digest_params_t *params, 475 void *op, 476 void **prep_req) 477 { 478 struct cpt_request_info *req; 479 uint32_t size, i; 480 uint16_t data_len, mac_len, key_len; 481 auth_type_t hash_type; 482 buf_ptr_t *meta_p; 483 struct cpt_ctx *ctx; 484 sg_comp_t *gather_comp; 485 sg_comp_t *scatter_comp; 486 uint8_t *in_buffer; 487 uint32_t g_size_bytes, s_size_bytes; 488 uint64_t dptr_dma, rptr_dma; 489 vq_cmd_word0_t vq_cmd_w0; 490 void *c_vaddr, *m_vaddr; 491 uint64_t c_dma, m_dma; 492 493 ctx = params->ctx_buf.vaddr; 494 meta_p = ¶ms->meta_buf; 495 496 m_vaddr = meta_p->vaddr; 497 m_dma = meta_p->dma_addr; 498 499 /* 500 * Save initial space that followed app data for completion code & 501 * alternate completion code to fall in same cache line as app data 502 */ 503 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 504 m_dma += COMPLETION_CODE_SIZE; 505 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 506 (uint8_t *)m_vaddr; 507 c_vaddr = (uint8_t *)m_vaddr + size; 508 c_dma = m_dma + size; 509 size += sizeof(cpt_res_s_t); 510 511 m_vaddr = (uint8_t *)m_vaddr + size; 512 m_dma += size; 513 514 req = m_vaddr; 515 516 size = sizeof(struct cpt_request_info); 517 m_vaddr = (uint8_t *)m_vaddr + size; 518 m_dma += size; 519 520 hash_type = ctx->hash_type; 521 mac_len = ctx->mac_len; 522 key_len = ctx->auth_key_len; 523 data_len = AUTH_DLEN(d_lens); 524 525 /*GP op header */ 526 vq_cmd_w0.s.opcode.minor = 0; 527 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 528 if (ctx->hmac) { 529 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 530 vq_cmd_w0.s.param1 = key_len; 531 vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8); 532 } else { 533 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 534 vq_cmd_w0.s.param1 = 0; 535 vq_cmd_w0.s.dlen = data_len; 536 } 537 538 /* Null auth only case enters the if */ 539 if (unlikely(!hash_type && !ctx->enc_cipher)) { 540 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC; 541 /* Minor op is passthrough */ 542 vq_cmd_w0.s.opcode.minor = 0x03; 543 /* Send out completion code only */ 544 vq_cmd_w0.s.param2 = 0x1; 545 } 546 547 /* DPTR has SG list */ 548 in_buffer = m_vaddr; 549 dptr_dma = m_dma; 550 551 ((uint16_t *)in_buffer)[0] = 0; 552 ((uint16_t *)in_buffer)[1] = 0; 553 554 /* TODO Add error check if space will be sufficient */ 555 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 556 557 /* 558 * Input gather list 559 */ 560 561 i = 0; 562 563 if (ctx->hmac) { 564 uint64_t k_dma = ctx->auth_key_iova; 565 /* Key */ 566 i = fill_sg_comp(gather_comp, i, k_dma, 567 RTE_ALIGN_CEIL(key_len, 8)); 568 } 569 570 /* input data */ 571 size = data_len; 572 if (size) { 573 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 574 0, &size, NULL, 0); 575 if (unlikely(size)) { 576 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 577 " by %dB", size); 578 return; 579 } 580 } else { 581 /* 582 * Looks like we need to support zero data 583 * gather ptr in case of hash & hmac 584 */ 585 i++; 586 } 587 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 588 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 589 590 /* 591 * Output Gather list 592 */ 593 594 i = 0; 595 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 596 597 if (flags & VALID_MAC_BUF) { 598 if (unlikely(params->mac_buf.size < mac_len)) { 599 CPT_LOG_DP_ERR("Insufficient MAC size"); 600 return; 601 } 602 603 size = mac_len; 604 i = fill_sg_comp_from_buf_min(scatter_comp, i, 605 ¶ms->mac_buf, &size); 606 } else { 607 size = mac_len; 608 i = fill_sg_comp_from_iov(scatter_comp, i, 609 params->src_iov, data_len, 610 &size, NULL, 0); 611 if (unlikely(size)) { 612 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 613 " %dB", size); 614 return; 615 } 616 } 617 618 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 619 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 620 621 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 622 623 /* This is DPTR len incase of SG mode */ 624 vq_cmd_w0.s.dlen = size; 625 626 m_vaddr = (uint8_t *)m_vaddr + size; 627 m_dma += size; 628 629 /* cpt alternate completion address saved earlier */ 630 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 631 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 632 rptr_dma = c_dma - 8; 633 634 req->ist.ei1 = dptr_dma; 635 req->ist.ei2 = rptr_dma; 636 637 /* 16 byte aligned cpt res address */ 638 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 639 *req->completion_addr = COMPLETION_CODE_INIT; 640 req->comp_baddr = c_dma; 641 642 /* Fill microcode part of instruction */ 643 req->ist.ei0 = vq_cmd_w0.u64; 644 645 req->op = op; 646 647 *prep_req = req; 648 return; 649 } 650 651 static __rte_always_inline void 652 cpt_enc_hmac_prep(uint32_t flags, 653 uint64_t d_offs, 654 uint64_t d_lens, 655 fc_params_t *fc_params, 656 void *op, 657 void **prep_req) 658 { 659 uint32_t iv_offset = 0; 660 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 661 struct cpt_ctx *cpt_ctx; 662 uint32_t cipher_type, hash_type; 663 uint32_t mac_len, size; 664 uint8_t iv_len = 16; 665 struct cpt_request_info *req; 666 buf_ptr_t *meta_p, *aad_buf = NULL; 667 uint32_t encr_offset, auth_offset; 668 uint32_t encr_data_len, auth_data_len, aad_len = 0; 669 uint32_t passthrough_len = 0; 670 void *m_vaddr, *offset_vaddr; 671 uint64_t m_dma, offset_dma; 672 vq_cmd_word0_t vq_cmd_w0; 673 void *c_vaddr; 674 uint64_t c_dma; 675 676 meta_p = &fc_params->meta_buf; 677 m_vaddr = meta_p->vaddr; 678 m_dma = meta_p->dma_addr; 679 680 encr_offset = ENCR_OFFSET(d_offs); 681 auth_offset = AUTH_OFFSET(d_offs); 682 encr_data_len = ENCR_DLEN(d_lens); 683 auth_data_len = AUTH_DLEN(d_lens); 684 if (unlikely(flags & VALID_AAD_BUF)) { 685 /* 686 * We dont support both aad 687 * and auth data separately 688 */ 689 auth_data_len = 0; 690 auth_offset = 0; 691 aad_len = fc_params->aad_buf.size; 692 aad_buf = &fc_params->aad_buf; 693 } 694 cpt_ctx = fc_params->ctx_buf.vaddr; 695 cipher_type = cpt_ctx->enc_cipher; 696 hash_type = cpt_ctx->hash_type; 697 mac_len = cpt_ctx->mac_len; 698 699 /* 700 * Save initial space that followed app data for completion code & 701 * alternate completion code to fall in same cache line as app data 702 */ 703 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 704 m_dma += COMPLETION_CODE_SIZE; 705 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 706 (uint8_t *)m_vaddr; 707 708 c_vaddr = (uint8_t *)m_vaddr + size; 709 c_dma = m_dma + size; 710 size += sizeof(cpt_res_s_t); 711 712 m_vaddr = (uint8_t *)m_vaddr + size; 713 m_dma += size; 714 715 /* start cpt request info struct at 8 byte boundary */ 716 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 717 (uint8_t *)m_vaddr; 718 719 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 720 721 size += sizeof(struct cpt_request_info); 722 m_vaddr = (uint8_t *)m_vaddr + size; 723 m_dma += size; 724 725 if (unlikely(!(flags & VALID_IV_BUF))) { 726 iv_len = 0; 727 iv_offset = ENCR_IV_OFFSET(d_offs); 728 } 729 730 if (unlikely(flags & VALID_AAD_BUF)) { 731 /* 732 * When AAD is given, data above encr_offset is pass through 733 * Since AAD is given as separate pointer and not as offset, 734 * this is a special case as we need to fragment input data 735 * into passthrough + encr_data and then insert AAD in between. 736 */ 737 if (hash_type != GMAC_TYPE) { 738 passthrough_len = encr_offset; 739 auth_offset = passthrough_len + iv_len; 740 encr_offset = passthrough_len + aad_len + iv_len; 741 auth_data_len = aad_len + encr_data_len; 742 } else { 743 passthrough_len = 16 + aad_len; 744 auth_offset = passthrough_len + iv_len; 745 auth_data_len = aad_len; 746 } 747 } else { 748 encr_offset += iv_len; 749 auth_offset += iv_len; 750 } 751 752 /* Encryption */ 753 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 754 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT; 755 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc << 756 CPT_HMAC_FIRST_BIT_POS); 757 758 if (hash_type == GMAC_TYPE) { 759 encr_offset = 0; 760 encr_data_len = 0; 761 } 762 763 auth_dlen = auth_offset + auth_data_len; 764 enc_dlen = encr_data_len + encr_offset; 765 if (unlikely(encr_data_len & 0xf)) { 766 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 767 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 768 encr_offset; 769 else if (likely((cipher_type == AES_CBC) || 770 (cipher_type == AES_ECB))) 771 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 772 encr_offset; 773 } 774 775 if (unlikely(auth_dlen > enc_dlen)) { 776 inputlen = auth_dlen; 777 outputlen = auth_dlen + mac_len; 778 } else { 779 inputlen = enc_dlen; 780 outputlen = enc_dlen + mac_len; 781 } 782 783 if (cpt_ctx->auth_enc != 0) 784 outputlen = enc_dlen; 785 786 /* GP op header */ 787 vq_cmd_w0.s.param1 = encr_data_len; 788 vq_cmd_w0.s.param2 = auth_data_len; 789 /* 790 * In 83XX since we have a limitation of 791 * IV & Offset control word not part of instruction 792 * and need to be part of Data Buffer, we check if 793 * head room is there and then only do the Direct mode processing 794 */ 795 if (likely((flags & SINGLE_BUF_INPLACE) && 796 (flags & SINGLE_BUF_HEADTAILROOM))) { 797 void *dm_vaddr = fc_params->bufs[0].vaddr; 798 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 799 /* 800 * This flag indicates that there is 24 bytes head room and 801 * 8 bytes tail room available, so that we get to do 802 * DIRECT MODE with limitation 803 */ 804 805 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 806 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 807 808 /* DPTR */ 809 req->ist.ei1 = offset_dma; 810 /* RPTR should just exclude offset control word */ 811 req->ist.ei2 = dm_dma_addr - iv_len; 812 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 813 + outputlen - iv_len); 814 815 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 816 817 if (likely(iv_len)) { 818 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 819 + OFF_CTRL_LEN); 820 uint64_t *src = fc_params->iv_buf; 821 dest[0] = src[0]; 822 dest[1] = src[1]; 823 } 824 825 *(uint64_t *)offset_vaddr = 826 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 827 ((uint64_t)iv_offset << 8) | 828 ((uint64_t)auth_offset)); 829 830 } else { 831 uint32_t i, g_size_bytes, s_size_bytes; 832 uint64_t dptr_dma, rptr_dma; 833 sg_comp_t *gather_comp; 834 sg_comp_t *scatter_comp; 835 uint8_t *in_buffer; 836 837 /* This falls under strict SG mode */ 838 offset_vaddr = m_vaddr; 839 offset_dma = m_dma; 840 size = OFF_CTRL_LEN + iv_len; 841 842 m_vaddr = (uint8_t *)m_vaddr + size; 843 m_dma += size; 844 845 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 846 847 if (likely(iv_len)) { 848 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 849 + OFF_CTRL_LEN); 850 uint64_t *src = fc_params->iv_buf; 851 dest[0] = src[0]; 852 dest[1] = src[1]; 853 } 854 855 *(uint64_t *)offset_vaddr = 856 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 857 ((uint64_t)iv_offset << 8) | 858 ((uint64_t)auth_offset)); 859 860 /* DPTR has SG list */ 861 in_buffer = m_vaddr; 862 dptr_dma = m_dma; 863 864 ((uint16_t *)in_buffer)[0] = 0; 865 ((uint16_t *)in_buffer)[1] = 0; 866 867 /* TODO Add error check if space will be sufficient */ 868 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 869 870 /* 871 * Input Gather List 872 */ 873 874 i = 0; 875 876 /* Offset control word that includes iv */ 877 i = fill_sg_comp(gather_comp, i, offset_dma, 878 OFF_CTRL_LEN + iv_len); 879 880 /* Add input data */ 881 size = inputlen - iv_len; 882 if (likely(size)) { 883 uint32_t aad_offset = aad_len ? passthrough_len : 0; 884 885 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 886 i = fill_sg_comp_from_buf_min(gather_comp, i, 887 fc_params->bufs, 888 &size); 889 } else { 890 i = fill_sg_comp_from_iov(gather_comp, i, 891 fc_params->src_iov, 892 0, &size, 893 aad_buf, aad_offset); 894 } 895 896 if (unlikely(size)) { 897 CPT_LOG_DP_ERR("Insufficient buffer space," 898 " size %d needed", size); 899 return; 900 } 901 } 902 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 903 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 904 905 /* 906 * Output Scatter list 907 */ 908 i = 0; 909 scatter_comp = 910 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 911 912 /* Add IV */ 913 if (likely(iv_len)) { 914 i = fill_sg_comp(scatter_comp, i, 915 offset_dma + OFF_CTRL_LEN, 916 iv_len); 917 } 918 919 /* output data or output data + digest*/ 920 if (unlikely(flags & VALID_MAC_BUF)) { 921 size = outputlen - iv_len - mac_len; 922 if (size) { 923 uint32_t aad_offset = 924 aad_len ? passthrough_len : 0; 925 926 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 927 i = fill_sg_comp_from_buf_min( 928 scatter_comp, 929 i, 930 fc_params->bufs, 931 &size); 932 } else { 933 i = fill_sg_comp_from_iov(scatter_comp, 934 i, 935 fc_params->dst_iov, 936 0, 937 &size, 938 aad_buf, 939 aad_offset); 940 } 941 if (unlikely(size)) { 942 CPT_LOG_DP_ERR("Insufficient buffer" 943 " space, size %d needed", 944 size); 945 return; 946 } 947 } 948 /* mac_data */ 949 if (mac_len) { 950 i = fill_sg_comp_from_buf(scatter_comp, i, 951 &fc_params->mac_buf); 952 } 953 } else { 954 /* Output including mac */ 955 size = outputlen - iv_len; 956 if (likely(size)) { 957 uint32_t aad_offset = 958 aad_len ? passthrough_len : 0; 959 960 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 961 i = fill_sg_comp_from_buf_min( 962 scatter_comp, 963 i, 964 fc_params->bufs, 965 &size); 966 } else { 967 i = fill_sg_comp_from_iov(scatter_comp, 968 i, 969 fc_params->dst_iov, 970 0, 971 &size, 972 aad_buf, 973 aad_offset); 974 } 975 if (unlikely(size)) { 976 CPT_LOG_DP_ERR("Insufficient buffer" 977 " space, size %d needed", 978 size); 979 return; 980 } 981 } 982 } 983 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 984 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 985 986 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 987 988 /* This is DPTR len incase of SG mode */ 989 vq_cmd_w0.s.dlen = size; 990 991 m_vaddr = (uint8_t *)m_vaddr + size; 992 m_dma += size; 993 994 /* cpt alternate completion address saved earlier */ 995 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 996 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 997 rptr_dma = c_dma - 8; 998 999 req->ist.ei1 = dptr_dma; 1000 req->ist.ei2 = rptr_dma; 1001 } 1002 1003 if (unlikely((encr_offset >> 16) || 1004 (iv_offset >> 8) || 1005 (auth_offset >> 8))) { 1006 CPT_LOG_DP_ERR("Offset not supported"); 1007 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1008 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1009 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1010 return; 1011 } 1012 1013 /* 16 byte aligned cpt res address */ 1014 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1015 *req->completion_addr = COMPLETION_CODE_INIT; 1016 req->comp_baddr = c_dma; 1017 1018 /* Fill microcode part of instruction */ 1019 req->ist.ei0 = vq_cmd_w0.u64; 1020 1021 req->op = op; 1022 1023 *prep_req = req; 1024 return; 1025 } 1026 1027 static __rte_always_inline void 1028 cpt_dec_hmac_prep(uint32_t flags, 1029 uint64_t d_offs, 1030 uint64_t d_lens, 1031 fc_params_t *fc_params, 1032 void *op, 1033 void **prep_req) 1034 { 1035 uint32_t iv_offset = 0, size; 1036 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1037 struct cpt_ctx *cpt_ctx; 1038 int32_t hash_type, mac_len; 1039 uint8_t iv_len = 16; 1040 struct cpt_request_info *req; 1041 buf_ptr_t *meta_p, *aad_buf = NULL; 1042 uint32_t encr_offset, auth_offset; 1043 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1044 uint32_t passthrough_len = 0; 1045 void *m_vaddr, *offset_vaddr; 1046 uint64_t m_dma, offset_dma; 1047 vq_cmd_word0_t vq_cmd_w0; 1048 void *c_vaddr; 1049 uint64_t c_dma; 1050 1051 meta_p = &fc_params->meta_buf; 1052 m_vaddr = meta_p->vaddr; 1053 m_dma = meta_p->dma_addr; 1054 1055 encr_offset = ENCR_OFFSET(d_offs); 1056 auth_offset = AUTH_OFFSET(d_offs); 1057 encr_data_len = ENCR_DLEN(d_lens); 1058 auth_data_len = AUTH_DLEN(d_lens); 1059 1060 if (unlikely(flags & VALID_AAD_BUF)) { 1061 /* 1062 * We dont support both aad 1063 * and auth data separately 1064 */ 1065 auth_data_len = 0; 1066 auth_offset = 0; 1067 aad_len = fc_params->aad_buf.size; 1068 aad_buf = &fc_params->aad_buf; 1069 } 1070 1071 cpt_ctx = fc_params->ctx_buf.vaddr; 1072 hash_type = cpt_ctx->hash_type; 1073 mac_len = cpt_ctx->mac_len; 1074 1075 if (unlikely(!(flags & VALID_IV_BUF))) { 1076 iv_len = 0; 1077 iv_offset = ENCR_IV_OFFSET(d_offs); 1078 } 1079 1080 if (unlikely(flags & VALID_AAD_BUF)) { 1081 /* 1082 * When AAD is given, data above encr_offset is pass through 1083 * Since AAD is given as separate pointer and not as offset, 1084 * this is a special case as we need to fragment input data 1085 * into passthrough + encr_data and then insert AAD in between. 1086 */ 1087 if (hash_type != GMAC_TYPE) { 1088 passthrough_len = encr_offset; 1089 auth_offset = passthrough_len + iv_len; 1090 encr_offset = passthrough_len + aad_len + iv_len; 1091 auth_data_len = aad_len + encr_data_len; 1092 } else { 1093 passthrough_len = 16 + aad_len; 1094 auth_offset = passthrough_len + iv_len; 1095 auth_data_len = aad_len; 1096 } 1097 } else { 1098 encr_offset += iv_len; 1099 auth_offset += iv_len; 1100 } 1101 1102 /* 1103 * Save initial space that followed app data for completion code & 1104 * alternate completion code to fall in same cache line as app data 1105 */ 1106 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1107 m_dma += COMPLETION_CODE_SIZE; 1108 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1109 (uint8_t *)m_vaddr; 1110 c_vaddr = (uint8_t *)m_vaddr + size; 1111 c_dma = m_dma + size; 1112 size += sizeof(cpt_res_s_t); 1113 1114 m_vaddr = (uint8_t *)m_vaddr + size; 1115 m_dma += size; 1116 1117 /* start cpt request info structure at 8 byte alignment */ 1118 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1119 (uint8_t *)m_vaddr; 1120 1121 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1122 1123 size += sizeof(struct cpt_request_info); 1124 m_vaddr = (uint8_t *)m_vaddr + size; 1125 m_dma += size; 1126 1127 /* Decryption */ 1128 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 1129 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT; 1130 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth << 1131 CPT_HMAC_FIRST_BIT_POS); 1132 1133 if (hash_type == GMAC_TYPE) { 1134 encr_offset = 0; 1135 encr_data_len = 0; 1136 } 1137 1138 enc_dlen = encr_offset + encr_data_len; 1139 auth_dlen = auth_offset + auth_data_len; 1140 1141 if (auth_dlen > enc_dlen) { 1142 inputlen = auth_dlen + mac_len; 1143 outputlen = auth_dlen; 1144 } else { 1145 inputlen = enc_dlen + mac_len; 1146 outputlen = enc_dlen; 1147 } 1148 1149 if (cpt_ctx->dec_auth != 0) 1150 outputlen = inputlen = enc_dlen; 1151 1152 vq_cmd_w0.s.param1 = encr_data_len; 1153 vq_cmd_w0.s.param2 = auth_data_len; 1154 1155 /* 1156 * In 83XX since we have a limitation of 1157 * IV & Offset control word not part of instruction 1158 * and need to be part of Data Buffer, we check if 1159 * head room is there and then only do the Direct mode processing 1160 */ 1161 if (likely((flags & SINGLE_BUF_INPLACE) && 1162 (flags & SINGLE_BUF_HEADTAILROOM))) { 1163 void *dm_vaddr = fc_params->bufs[0].vaddr; 1164 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1165 /* 1166 * This flag indicates that there is 24 bytes head room and 1167 * 8 bytes tail room available, so that we get to do 1168 * DIRECT MODE with limitation 1169 */ 1170 1171 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1172 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1173 req->ist.ei1 = offset_dma; 1174 1175 /* RPTR should just exclude offset control word */ 1176 req->ist.ei2 = dm_dma_addr - iv_len; 1177 1178 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1179 outputlen - iv_len); 1180 /* since this is decryption, 1181 * don't touch the content of 1182 * alternate ccode space as it contains 1183 * hmac. 1184 */ 1185 1186 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1187 1188 if (likely(iv_len)) { 1189 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1190 OFF_CTRL_LEN); 1191 uint64_t *src = fc_params->iv_buf; 1192 dest[0] = src[0]; 1193 dest[1] = src[1]; 1194 } 1195 1196 if (unlikely((encr_offset >> 16) || 1197 (iv_offset >> 8) || 1198 (auth_offset >> 8))) { 1199 CPT_LOG_DP_ERR("Offset not supported"); 1200 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1201 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1202 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1203 return; 1204 } 1205 1206 *(uint64_t *)offset_vaddr = 1207 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1208 ((uint64_t)iv_offset << 8) | 1209 ((uint64_t)auth_offset)); 1210 1211 } else { 1212 uint64_t dptr_dma, rptr_dma; 1213 uint32_t g_size_bytes, s_size_bytes; 1214 sg_comp_t *gather_comp; 1215 sg_comp_t *scatter_comp; 1216 uint8_t *in_buffer; 1217 uint8_t i = 0; 1218 1219 /* This falls under strict SG mode */ 1220 offset_vaddr = m_vaddr; 1221 offset_dma = m_dma; 1222 size = OFF_CTRL_LEN + iv_len; 1223 1224 m_vaddr = (uint8_t *)m_vaddr + size; 1225 m_dma += size; 1226 1227 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1228 1229 if (likely(iv_len)) { 1230 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1231 OFF_CTRL_LEN); 1232 uint64_t *src = fc_params->iv_buf; 1233 dest[0] = src[0]; 1234 dest[1] = src[1]; 1235 } 1236 1237 if (unlikely((encr_offset >> 16) || 1238 (iv_offset >> 8) || 1239 (auth_offset >> 8))) { 1240 CPT_LOG_DP_ERR("Offset not supported"); 1241 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1242 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1243 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1244 return; 1245 } 1246 1247 *(uint64_t *)offset_vaddr = 1248 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1249 ((uint64_t)iv_offset << 8) | 1250 ((uint64_t)auth_offset)); 1251 1252 /* DPTR has SG list */ 1253 in_buffer = m_vaddr; 1254 dptr_dma = m_dma; 1255 1256 ((uint16_t *)in_buffer)[0] = 0; 1257 ((uint16_t *)in_buffer)[1] = 0; 1258 1259 /* TODO Add error check if space will be sufficient */ 1260 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1261 1262 /* 1263 * Input Gather List 1264 */ 1265 i = 0; 1266 1267 /* Offset control word that includes iv */ 1268 i = fill_sg_comp(gather_comp, i, offset_dma, 1269 OFF_CTRL_LEN + iv_len); 1270 1271 /* Add input data */ 1272 if (flags & VALID_MAC_BUF) { 1273 size = inputlen - iv_len - mac_len; 1274 if (size) { 1275 /* input data only */ 1276 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1277 i = fill_sg_comp_from_buf_min( 1278 gather_comp, i, 1279 fc_params->bufs, 1280 &size); 1281 } else { 1282 uint32_t aad_offset = aad_len ? 1283 passthrough_len : 0; 1284 1285 i = fill_sg_comp_from_iov(gather_comp, 1286 i, 1287 fc_params->src_iov, 1288 0, &size, 1289 aad_buf, 1290 aad_offset); 1291 } 1292 if (unlikely(size)) { 1293 CPT_LOG_DP_ERR("Insufficient buffer" 1294 " space, size %d needed", 1295 size); 1296 return; 1297 } 1298 } 1299 1300 /* mac data */ 1301 if (mac_len) { 1302 i = fill_sg_comp_from_buf(gather_comp, i, 1303 &fc_params->mac_buf); 1304 } 1305 } else { 1306 /* input data + mac */ 1307 size = inputlen - iv_len; 1308 if (size) { 1309 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1310 i = fill_sg_comp_from_buf_min( 1311 gather_comp, i, 1312 fc_params->bufs, 1313 &size); 1314 } else { 1315 uint32_t aad_offset = aad_len ? 1316 passthrough_len : 0; 1317 1318 if (unlikely(!fc_params->src_iov)) { 1319 CPT_LOG_DP_ERR("Bad input args"); 1320 return; 1321 } 1322 1323 i = fill_sg_comp_from_iov( 1324 gather_comp, i, 1325 fc_params->src_iov, 1326 0, &size, 1327 aad_buf, 1328 aad_offset); 1329 } 1330 1331 if (unlikely(size)) { 1332 CPT_LOG_DP_ERR("Insufficient buffer" 1333 " space, size %d needed", 1334 size); 1335 return; 1336 } 1337 } 1338 } 1339 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1340 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1341 1342 /* 1343 * Output Scatter List 1344 */ 1345 1346 i = 0; 1347 scatter_comp = 1348 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1349 1350 /* Add iv */ 1351 if (iv_len) { 1352 i = fill_sg_comp(scatter_comp, i, 1353 offset_dma + OFF_CTRL_LEN, 1354 iv_len); 1355 } 1356 1357 /* Add output data */ 1358 size = outputlen - iv_len; 1359 if (size) { 1360 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1361 /* handle single buffer here */ 1362 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1363 fc_params->bufs, 1364 &size); 1365 } else { 1366 uint32_t aad_offset = aad_len ? 1367 passthrough_len : 0; 1368 1369 if (unlikely(!fc_params->dst_iov)) { 1370 CPT_LOG_DP_ERR("Bad input args"); 1371 return; 1372 } 1373 1374 i = fill_sg_comp_from_iov(scatter_comp, i, 1375 fc_params->dst_iov, 0, 1376 &size, aad_buf, 1377 aad_offset); 1378 } 1379 1380 if (unlikely(size)) { 1381 CPT_LOG_DP_ERR("Insufficient buffer space," 1382 " size %d needed", size); 1383 return; 1384 } 1385 } 1386 1387 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1388 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1389 1390 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1391 1392 /* This is DPTR len incase of SG mode */ 1393 vq_cmd_w0.s.dlen = size; 1394 1395 m_vaddr = (uint8_t *)m_vaddr + size; 1396 m_dma += size; 1397 1398 /* cpt alternate completion address saved earlier */ 1399 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1400 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1401 rptr_dma = c_dma - 8; 1402 size += COMPLETION_CODE_SIZE; 1403 1404 req->ist.ei1 = dptr_dma; 1405 req->ist.ei2 = rptr_dma; 1406 } 1407 1408 /* 16 byte aligned cpt res address */ 1409 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1410 *req->completion_addr = COMPLETION_CODE_INIT; 1411 req->comp_baddr = c_dma; 1412 1413 /* Fill microcode part of instruction */ 1414 req->ist.ei0 = vq_cmd_w0.u64; 1415 1416 req->op = op; 1417 1418 *prep_req = req; 1419 return; 1420 } 1421 1422 static __rte_always_inline void 1423 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1424 uint64_t d_offs, 1425 uint64_t d_lens, 1426 fc_params_t *params, 1427 void *op, 1428 void **prep_req) 1429 { 1430 uint32_t size; 1431 int32_t inputlen, outputlen; 1432 struct cpt_ctx *cpt_ctx; 1433 uint32_t mac_len = 0; 1434 uint8_t snow3g, j; 1435 struct cpt_request_info *req; 1436 buf_ptr_t *buf_p; 1437 uint32_t encr_offset = 0, auth_offset = 0; 1438 uint32_t encr_data_len = 0, auth_data_len = 0; 1439 int flags, iv_len = 16; 1440 void *m_vaddr, *c_vaddr; 1441 uint64_t m_dma, c_dma, offset_ctrl; 1442 uint64_t *offset_vaddr, offset_dma; 1443 uint32_t *iv_s, iv[4]; 1444 vq_cmd_word0_t vq_cmd_w0; 1445 1446 buf_p = ¶ms->meta_buf; 1447 m_vaddr = buf_p->vaddr; 1448 m_dma = buf_p->dma_addr; 1449 1450 cpt_ctx = params->ctx_buf.vaddr; 1451 flags = cpt_ctx->zsk_flags; 1452 mac_len = cpt_ctx->mac_len; 1453 snow3g = cpt_ctx->snow3g; 1454 1455 /* 1456 * Save initial space that followed app data for completion code & 1457 * alternate completion code to fall in same cache line as app data 1458 */ 1459 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1460 m_dma += COMPLETION_CODE_SIZE; 1461 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1462 (uint8_t *)m_vaddr; 1463 1464 c_vaddr = (uint8_t *)m_vaddr + size; 1465 c_dma = m_dma + size; 1466 size += sizeof(cpt_res_s_t); 1467 1468 m_vaddr = (uint8_t *)m_vaddr + size; 1469 m_dma += size; 1470 1471 /* Reserve memory for cpt request info */ 1472 req = m_vaddr; 1473 1474 size = sizeof(struct cpt_request_info); 1475 m_vaddr = (uint8_t *)m_vaddr + size; 1476 m_dma += size; 1477 1478 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1479 1480 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1481 1482 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1483 (0 << 3) | (flags & 0x7)); 1484 1485 if (flags == 0x1) { 1486 /* 1487 * Microcode expects offsets in bytes 1488 * TODO: Rounding off 1489 */ 1490 auth_data_len = AUTH_DLEN(d_lens); 1491 1492 /* EIA3 or UIA2 */ 1493 auth_offset = AUTH_OFFSET(d_offs); 1494 auth_offset = auth_offset / 8; 1495 1496 /* consider iv len */ 1497 auth_offset += iv_len; 1498 1499 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1500 outputlen = mac_len; 1501 1502 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1503 1504 } else { 1505 /* EEA3 or UEA2 */ 1506 /* 1507 * Microcode expects offsets in bytes 1508 * TODO: Rounding off 1509 */ 1510 encr_data_len = ENCR_DLEN(d_lens); 1511 1512 encr_offset = ENCR_OFFSET(d_offs); 1513 encr_offset = encr_offset / 8; 1514 /* consider iv len */ 1515 encr_offset += iv_len; 1516 1517 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1518 outputlen = inputlen; 1519 1520 /* iv offset is 0 */ 1521 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1522 } 1523 1524 if (unlikely((encr_offset >> 16) || 1525 (auth_offset >> 8))) { 1526 CPT_LOG_DP_ERR("Offset not supported"); 1527 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1528 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1529 return; 1530 } 1531 1532 /* IV */ 1533 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1534 params->iv_buf; 1535 1536 if (snow3g) { 1537 /* 1538 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1539 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1540 */ 1541 1542 for (j = 0; j < 4; j++) 1543 iv[j] = iv_s[3 - j]; 1544 } else { 1545 /* ZUC doesn't need a swap */ 1546 for (j = 0; j < 4; j++) 1547 iv[j] = iv_s[j]; 1548 } 1549 1550 /* 1551 * GP op header, lengths are expected in bits. 1552 */ 1553 vq_cmd_w0.s.param1 = encr_data_len; 1554 vq_cmd_w0.s.param2 = auth_data_len; 1555 1556 /* 1557 * In 83XX since we have a limitation of 1558 * IV & Offset control word not part of instruction 1559 * and need to be part of Data Buffer, we check if 1560 * head room is there and then only do the Direct mode processing 1561 */ 1562 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1563 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1564 void *dm_vaddr = params->bufs[0].vaddr; 1565 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1566 /* 1567 * This flag indicates that there is 24 bytes head room and 1568 * 8 bytes tail room available, so that we get to do 1569 * DIRECT MODE with limitation 1570 */ 1571 1572 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1573 OFF_CTRL_LEN - iv_len); 1574 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1575 1576 /* DPTR */ 1577 req->ist.ei1 = offset_dma; 1578 /* RPTR should just exclude offset control word */ 1579 req->ist.ei2 = dm_dma_addr - iv_len; 1580 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1581 + outputlen - iv_len); 1582 1583 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1584 1585 if (likely(iv_len)) { 1586 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1587 + OFF_CTRL_LEN); 1588 memcpy(iv_d, iv, 16); 1589 } 1590 1591 *offset_vaddr = offset_ctrl; 1592 } else { 1593 uint32_t i, g_size_bytes, s_size_bytes; 1594 uint64_t dptr_dma, rptr_dma; 1595 sg_comp_t *gather_comp; 1596 sg_comp_t *scatter_comp; 1597 uint8_t *in_buffer; 1598 uint32_t *iv_d; 1599 1600 /* save space for iv */ 1601 offset_vaddr = m_vaddr; 1602 offset_dma = m_dma; 1603 1604 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1605 m_dma += OFF_CTRL_LEN + iv_len; 1606 1607 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1608 1609 /* DPTR has SG list */ 1610 in_buffer = m_vaddr; 1611 dptr_dma = m_dma; 1612 1613 ((uint16_t *)in_buffer)[0] = 0; 1614 ((uint16_t *)in_buffer)[1] = 0; 1615 1616 /* TODO Add error check if space will be sufficient */ 1617 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1618 1619 /* 1620 * Input Gather List 1621 */ 1622 i = 0; 1623 1624 /* Offset control word followed by iv */ 1625 1626 i = fill_sg_comp(gather_comp, i, offset_dma, 1627 OFF_CTRL_LEN + iv_len); 1628 1629 /* iv offset is 0 */ 1630 *offset_vaddr = offset_ctrl; 1631 1632 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1633 memcpy(iv_d, iv, 16); 1634 1635 /* input data */ 1636 size = inputlen - iv_len; 1637 if (size) { 1638 i = fill_sg_comp_from_iov(gather_comp, i, 1639 params->src_iov, 1640 0, &size, NULL, 0); 1641 if (unlikely(size)) { 1642 CPT_LOG_DP_ERR("Insufficient buffer space," 1643 " size %d needed", size); 1644 return; 1645 } 1646 } 1647 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1648 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1649 1650 /* 1651 * Output Scatter List 1652 */ 1653 1654 i = 0; 1655 scatter_comp = 1656 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1657 1658 if (flags == 0x1) { 1659 /* IV in SLIST only for EEA3 & UEA2 */ 1660 iv_len = 0; 1661 } 1662 1663 if (iv_len) { 1664 i = fill_sg_comp(scatter_comp, i, 1665 offset_dma + OFF_CTRL_LEN, iv_len); 1666 } 1667 1668 /* Add output data */ 1669 if (req_flags & VALID_MAC_BUF) { 1670 size = outputlen - iv_len - mac_len; 1671 if (size) { 1672 i = fill_sg_comp_from_iov(scatter_comp, i, 1673 params->dst_iov, 0, 1674 &size, NULL, 0); 1675 1676 if (unlikely(size)) { 1677 CPT_LOG_DP_ERR("Insufficient buffer space," 1678 " size %d needed", size); 1679 return; 1680 } 1681 } 1682 1683 /* mac data */ 1684 if (mac_len) { 1685 i = fill_sg_comp_from_buf(scatter_comp, i, 1686 ¶ms->mac_buf); 1687 } 1688 } else { 1689 /* Output including mac */ 1690 size = outputlen - iv_len; 1691 if (size) { 1692 i = fill_sg_comp_from_iov(scatter_comp, i, 1693 params->dst_iov, 0, 1694 &size, NULL, 0); 1695 1696 if (unlikely(size)) { 1697 CPT_LOG_DP_ERR("Insufficient buffer space," 1698 " size %d needed", size); 1699 return; 1700 } 1701 } 1702 } 1703 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1704 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1705 1706 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1707 1708 /* This is DPTR len incase of SG mode */ 1709 vq_cmd_w0.s.dlen = size; 1710 1711 m_vaddr = (uint8_t *)m_vaddr + size; 1712 m_dma += size; 1713 1714 /* cpt alternate completion address saved earlier */ 1715 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1716 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1717 rptr_dma = c_dma - 8; 1718 1719 req->ist.ei1 = dptr_dma; 1720 req->ist.ei2 = rptr_dma; 1721 } 1722 1723 /* 16 byte aligned cpt res address */ 1724 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1725 *req->completion_addr = COMPLETION_CODE_INIT; 1726 req->comp_baddr = c_dma; 1727 1728 /* Fill microcode part of instruction */ 1729 req->ist.ei0 = vq_cmd_w0.u64; 1730 1731 req->op = op; 1732 1733 *prep_req = req; 1734 return; 1735 } 1736 1737 static __rte_always_inline void 1738 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1739 uint64_t d_offs, 1740 uint64_t d_lens, 1741 fc_params_t *params, 1742 void *op, 1743 void **prep_req) 1744 { 1745 uint32_t size; 1746 int32_t inputlen = 0, outputlen; 1747 struct cpt_ctx *cpt_ctx; 1748 uint8_t snow3g, iv_len = 16; 1749 struct cpt_request_info *req; 1750 buf_ptr_t *buf_p; 1751 uint32_t encr_offset; 1752 uint32_t encr_data_len; 1753 int flags; 1754 void *m_vaddr, *c_vaddr; 1755 uint64_t m_dma, c_dma; 1756 uint64_t *offset_vaddr, offset_dma; 1757 uint32_t *iv_s, iv[4], j; 1758 vq_cmd_word0_t vq_cmd_w0; 1759 1760 buf_p = ¶ms->meta_buf; 1761 m_vaddr = buf_p->vaddr; 1762 m_dma = buf_p->dma_addr; 1763 1764 /* 1765 * Microcode expects offsets in bytes 1766 * TODO: Rounding off 1767 */ 1768 encr_offset = ENCR_OFFSET(d_offs) / 8; 1769 encr_data_len = ENCR_DLEN(d_lens); 1770 1771 cpt_ctx = params->ctx_buf.vaddr; 1772 flags = cpt_ctx->zsk_flags; 1773 snow3g = cpt_ctx->snow3g; 1774 /* 1775 * Save initial space that followed app data for completion code & 1776 * alternate completion code to fall in same cache line as app data 1777 */ 1778 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1779 m_dma += COMPLETION_CODE_SIZE; 1780 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1781 (uint8_t *)m_vaddr; 1782 1783 c_vaddr = (uint8_t *)m_vaddr + size; 1784 c_dma = m_dma + size; 1785 size += sizeof(cpt_res_s_t); 1786 1787 m_vaddr = (uint8_t *)m_vaddr + size; 1788 m_dma += size; 1789 1790 /* Reserve memory for cpt request info */ 1791 req = m_vaddr; 1792 1793 size = sizeof(struct cpt_request_info); 1794 m_vaddr = (uint8_t *)m_vaddr + size; 1795 m_dma += size; 1796 1797 vq_cmd_w0.u64 = 0; 1798 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1799 1800 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1801 1802 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1803 (0 << 3) | (flags & 0x7)); 1804 1805 /* consider iv len */ 1806 encr_offset += iv_len; 1807 1808 inputlen = encr_offset + 1809 (RTE_ALIGN(encr_data_len, 8) / 8); 1810 outputlen = inputlen; 1811 1812 /* IV */ 1813 iv_s = params->iv_buf; 1814 if (snow3g) { 1815 /* 1816 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1817 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1818 */ 1819 1820 for (j = 0; j < 4; j++) 1821 iv[j] = iv_s[3 - j]; 1822 } else { 1823 /* ZUC doesn't need a swap */ 1824 for (j = 0; j < 4; j++) 1825 iv[j] = iv_s[j]; 1826 } 1827 1828 /* 1829 * GP op header, lengths are expected in bits. 1830 */ 1831 vq_cmd_w0.s.param1 = encr_data_len; 1832 1833 /* 1834 * In 83XX since we have a limitation of 1835 * IV & Offset control word not part of instruction 1836 * and need to be part of Data Buffer, we check if 1837 * head room is there and then only do the Direct mode processing 1838 */ 1839 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1840 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1841 void *dm_vaddr = params->bufs[0].vaddr; 1842 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1843 /* 1844 * This flag indicates that there is 24 bytes head room and 1845 * 8 bytes tail room available, so that we get to do 1846 * DIRECT MODE with limitation 1847 */ 1848 1849 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1850 OFF_CTRL_LEN - iv_len); 1851 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1852 1853 /* DPTR */ 1854 req->ist.ei1 = offset_dma; 1855 /* RPTR should just exclude offset control word */ 1856 req->ist.ei2 = dm_dma_addr - iv_len; 1857 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1858 + outputlen - iv_len); 1859 1860 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1861 1862 if (likely(iv_len)) { 1863 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1864 + OFF_CTRL_LEN); 1865 memcpy(iv_d, iv, 16); 1866 } 1867 1868 /* iv offset is 0 */ 1869 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1870 } else { 1871 uint32_t i, g_size_bytes, s_size_bytes; 1872 uint64_t dptr_dma, rptr_dma; 1873 sg_comp_t *gather_comp; 1874 sg_comp_t *scatter_comp; 1875 uint8_t *in_buffer; 1876 uint32_t *iv_d; 1877 1878 /* save space for offset and iv... */ 1879 offset_vaddr = m_vaddr; 1880 offset_dma = m_dma; 1881 1882 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1883 m_dma += OFF_CTRL_LEN + iv_len; 1884 1885 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1886 1887 /* DPTR has SG list */ 1888 in_buffer = m_vaddr; 1889 dptr_dma = m_dma; 1890 1891 ((uint16_t *)in_buffer)[0] = 0; 1892 ((uint16_t *)in_buffer)[1] = 0; 1893 1894 /* TODO Add error check if space will be sufficient */ 1895 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1896 1897 /* 1898 * Input Gather List 1899 */ 1900 i = 0; 1901 1902 /* Offset control word */ 1903 1904 /* iv offset is 0 */ 1905 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1906 1907 i = fill_sg_comp(gather_comp, i, offset_dma, 1908 OFF_CTRL_LEN + iv_len); 1909 1910 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1911 memcpy(iv_d, iv, 16); 1912 1913 /* Add input data */ 1914 size = inputlen - iv_len; 1915 if (size) { 1916 i = fill_sg_comp_from_iov(gather_comp, i, 1917 params->src_iov, 1918 0, &size, NULL, 0); 1919 if (unlikely(size)) { 1920 CPT_LOG_DP_ERR("Insufficient buffer space," 1921 " size %d needed", size); 1922 return; 1923 } 1924 } 1925 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1926 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1927 1928 /* 1929 * Output Scatter List 1930 */ 1931 1932 i = 0; 1933 scatter_comp = 1934 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1935 1936 /* IV */ 1937 i = fill_sg_comp(scatter_comp, i, 1938 offset_dma + OFF_CTRL_LEN, 1939 iv_len); 1940 1941 /* Add output data */ 1942 size = outputlen - iv_len; 1943 if (size) { 1944 i = fill_sg_comp_from_iov(scatter_comp, i, 1945 params->dst_iov, 0, 1946 &size, NULL, 0); 1947 1948 if (unlikely(size)) { 1949 CPT_LOG_DP_ERR("Insufficient buffer space," 1950 " size %d needed", size); 1951 return; 1952 } 1953 } 1954 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1955 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1956 1957 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1958 1959 /* This is DPTR len incase of SG mode */ 1960 vq_cmd_w0.s.dlen = size; 1961 1962 m_vaddr = (uint8_t *)m_vaddr + size; 1963 m_dma += size; 1964 1965 /* cpt alternate completion address saved earlier */ 1966 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1967 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1968 rptr_dma = c_dma - 8; 1969 1970 req->ist.ei1 = dptr_dma; 1971 req->ist.ei2 = rptr_dma; 1972 } 1973 1974 if (unlikely((encr_offset >> 16))) { 1975 CPT_LOG_DP_ERR("Offset not supported"); 1976 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1977 return; 1978 } 1979 1980 /* 16 byte aligned cpt res address */ 1981 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1982 *req->completion_addr = COMPLETION_CODE_INIT; 1983 req->comp_baddr = c_dma; 1984 1985 /* Fill microcode part of instruction */ 1986 req->ist.ei0 = vq_cmd_w0.u64; 1987 1988 req->op = op; 1989 1990 *prep_req = req; 1991 return; 1992 } 1993 1994 static __rte_always_inline void 1995 cpt_kasumi_enc_prep(uint32_t req_flags, 1996 uint64_t d_offs, 1997 uint64_t d_lens, 1998 fc_params_t *params, 1999 void *op, 2000 void **prep_req) 2001 { 2002 uint32_t size; 2003 int32_t inputlen = 0, outputlen = 0; 2004 struct cpt_ctx *cpt_ctx; 2005 uint32_t mac_len = 0; 2006 uint8_t i = 0; 2007 struct cpt_request_info *req; 2008 buf_ptr_t *buf_p; 2009 uint32_t encr_offset, auth_offset; 2010 uint32_t encr_data_len, auth_data_len; 2011 int flags; 2012 uint8_t *iv_s, *iv_d, iv_len = 8; 2013 uint8_t dir = 0; 2014 void *m_vaddr, *c_vaddr; 2015 uint64_t m_dma, c_dma; 2016 uint64_t *offset_vaddr, offset_dma; 2017 vq_cmd_word0_t vq_cmd_w0; 2018 uint8_t *in_buffer; 2019 uint32_t g_size_bytes, s_size_bytes; 2020 uint64_t dptr_dma, rptr_dma; 2021 sg_comp_t *gather_comp; 2022 sg_comp_t *scatter_comp; 2023 2024 buf_p = ¶ms->meta_buf; 2025 m_vaddr = buf_p->vaddr; 2026 m_dma = buf_p->dma_addr; 2027 2028 encr_offset = ENCR_OFFSET(d_offs) / 8; 2029 auth_offset = AUTH_OFFSET(d_offs) / 8; 2030 encr_data_len = ENCR_DLEN(d_lens); 2031 auth_data_len = AUTH_DLEN(d_lens); 2032 2033 cpt_ctx = params->ctx_buf.vaddr; 2034 flags = cpt_ctx->zsk_flags; 2035 mac_len = cpt_ctx->mac_len; 2036 2037 if (flags == 0x0) 2038 iv_s = params->iv_buf; 2039 else 2040 iv_s = params->auth_iv_buf; 2041 2042 dir = iv_s[8] & 0x1; 2043 2044 /* 2045 * Save initial space that followed app data for completion code & 2046 * alternate completion code to fall in same cache line as app data 2047 */ 2048 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2049 m_dma += COMPLETION_CODE_SIZE; 2050 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2051 (uint8_t *)m_vaddr; 2052 2053 c_vaddr = (uint8_t *)m_vaddr + size; 2054 c_dma = m_dma + size; 2055 size += sizeof(cpt_res_s_t); 2056 2057 m_vaddr = (uint8_t *)m_vaddr + size; 2058 m_dma += size; 2059 2060 /* Reserve memory for cpt request info */ 2061 req = m_vaddr; 2062 2063 size = sizeof(struct cpt_request_info); 2064 m_vaddr = (uint8_t *)m_vaddr + size; 2065 m_dma += size; 2066 2067 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2068 2069 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2070 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2071 (dir << 4) | (0 << 3) | (flags & 0x7)); 2072 2073 /* 2074 * GP op header, lengths are expected in bits. 2075 */ 2076 vq_cmd_w0.s.param1 = encr_data_len; 2077 vq_cmd_w0.s.param2 = auth_data_len; 2078 2079 /* consider iv len */ 2080 if (flags == 0x0) { 2081 encr_offset += iv_len; 2082 auth_offset += iv_len; 2083 } 2084 2085 /* save space for offset ctrl and iv */ 2086 offset_vaddr = m_vaddr; 2087 offset_dma = m_dma; 2088 2089 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2090 m_dma += OFF_CTRL_LEN + iv_len; 2091 2092 /* DPTR has SG list */ 2093 in_buffer = m_vaddr; 2094 dptr_dma = m_dma; 2095 2096 ((uint16_t *)in_buffer)[0] = 0; 2097 ((uint16_t *)in_buffer)[1] = 0; 2098 2099 /* TODO Add error check if space will be sufficient */ 2100 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2101 2102 /* 2103 * Input Gather List 2104 */ 2105 i = 0; 2106 2107 /* Offset control word followed by iv */ 2108 2109 if (flags == 0x0) { 2110 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2111 outputlen = inputlen; 2112 /* iv offset is 0 */ 2113 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2114 if (unlikely((encr_offset >> 16))) { 2115 CPT_LOG_DP_ERR("Offset not supported"); 2116 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 2117 return; 2118 } 2119 } else { 2120 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2121 outputlen = mac_len; 2122 /* iv offset is 0 */ 2123 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2124 if (unlikely((auth_offset >> 8))) { 2125 CPT_LOG_DP_ERR("Offset not supported"); 2126 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 2127 return; 2128 } 2129 } 2130 2131 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2132 2133 /* IV */ 2134 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2135 memcpy(iv_d, iv_s, iv_len); 2136 2137 /* input data */ 2138 size = inputlen - iv_len; 2139 if (size) { 2140 i = fill_sg_comp_from_iov(gather_comp, i, 2141 params->src_iov, 0, 2142 &size, NULL, 0); 2143 2144 if (unlikely(size)) { 2145 CPT_LOG_DP_ERR("Insufficient buffer space," 2146 " size %d needed", size); 2147 return; 2148 } 2149 } 2150 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2151 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2152 2153 /* 2154 * Output Scatter List 2155 */ 2156 2157 i = 0; 2158 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2159 2160 if (flags == 0x1) { 2161 /* IV in SLIST only for F8 */ 2162 iv_len = 0; 2163 } 2164 2165 /* IV */ 2166 if (iv_len) { 2167 i = fill_sg_comp(scatter_comp, i, 2168 offset_dma + OFF_CTRL_LEN, 2169 iv_len); 2170 } 2171 2172 /* Add output data */ 2173 if (req_flags & VALID_MAC_BUF) { 2174 size = outputlen - iv_len - mac_len; 2175 if (size) { 2176 i = fill_sg_comp_from_iov(scatter_comp, i, 2177 params->dst_iov, 0, 2178 &size, NULL, 0); 2179 2180 if (unlikely(size)) { 2181 CPT_LOG_DP_ERR("Insufficient buffer space," 2182 " size %d needed", size); 2183 return; 2184 } 2185 } 2186 2187 /* mac data */ 2188 if (mac_len) { 2189 i = fill_sg_comp_from_buf(scatter_comp, i, 2190 ¶ms->mac_buf); 2191 } 2192 } else { 2193 /* Output including mac */ 2194 size = outputlen - iv_len; 2195 if (size) { 2196 i = fill_sg_comp_from_iov(scatter_comp, i, 2197 params->dst_iov, 0, 2198 &size, NULL, 0); 2199 2200 if (unlikely(size)) { 2201 CPT_LOG_DP_ERR("Insufficient buffer space," 2202 " size %d needed", size); 2203 return; 2204 } 2205 } 2206 } 2207 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2208 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2209 2210 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2211 2212 /* This is DPTR len incase of SG mode */ 2213 vq_cmd_w0.s.dlen = size; 2214 2215 m_vaddr = (uint8_t *)m_vaddr + size; 2216 m_dma += size; 2217 2218 /* cpt alternate completion address saved earlier */ 2219 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2220 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2221 rptr_dma = c_dma - 8; 2222 2223 req->ist.ei1 = dptr_dma; 2224 req->ist.ei2 = rptr_dma; 2225 2226 /* 16 byte aligned cpt res address */ 2227 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2228 *req->completion_addr = COMPLETION_CODE_INIT; 2229 req->comp_baddr = c_dma; 2230 2231 /* Fill microcode part of instruction */ 2232 req->ist.ei0 = vq_cmd_w0.u64; 2233 2234 req->op = op; 2235 2236 *prep_req = req; 2237 return; 2238 } 2239 2240 static __rte_always_inline void 2241 cpt_kasumi_dec_prep(uint64_t d_offs, 2242 uint64_t d_lens, 2243 fc_params_t *params, 2244 void *op, 2245 void **prep_req) 2246 { 2247 uint32_t size; 2248 int32_t inputlen = 0, outputlen; 2249 struct cpt_ctx *cpt_ctx; 2250 uint8_t i = 0, iv_len = 8; 2251 struct cpt_request_info *req; 2252 buf_ptr_t *buf_p; 2253 uint32_t encr_offset; 2254 uint32_t encr_data_len; 2255 int flags; 2256 uint8_t dir = 0; 2257 void *m_vaddr, *c_vaddr; 2258 uint64_t m_dma, c_dma; 2259 uint64_t *offset_vaddr, offset_dma; 2260 vq_cmd_word0_t vq_cmd_w0; 2261 uint8_t *in_buffer; 2262 uint32_t g_size_bytes, s_size_bytes; 2263 uint64_t dptr_dma, rptr_dma; 2264 sg_comp_t *gather_comp; 2265 sg_comp_t *scatter_comp; 2266 2267 buf_p = ¶ms->meta_buf; 2268 m_vaddr = buf_p->vaddr; 2269 m_dma = buf_p->dma_addr; 2270 2271 encr_offset = ENCR_OFFSET(d_offs) / 8; 2272 encr_data_len = ENCR_DLEN(d_lens); 2273 2274 cpt_ctx = params->ctx_buf.vaddr; 2275 flags = cpt_ctx->zsk_flags; 2276 /* 2277 * Save initial space that followed app data for completion code & 2278 * alternate completion code to fall in same cache line as app data 2279 */ 2280 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2281 m_dma += COMPLETION_CODE_SIZE; 2282 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2283 (uint8_t *)m_vaddr; 2284 2285 c_vaddr = (uint8_t *)m_vaddr + size; 2286 c_dma = m_dma + size; 2287 size += sizeof(cpt_res_s_t); 2288 2289 m_vaddr = (uint8_t *)m_vaddr + size; 2290 m_dma += size; 2291 2292 /* Reserve memory for cpt request info */ 2293 req = m_vaddr; 2294 2295 size = sizeof(struct cpt_request_info); 2296 m_vaddr = (uint8_t *)m_vaddr + size; 2297 m_dma += size; 2298 2299 vq_cmd_w0.u64 = 0; 2300 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2301 2302 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2303 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2304 (dir << 4) | (0 << 3) | (flags & 0x7)); 2305 2306 /* 2307 * GP op header, lengths are expected in bits. 2308 */ 2309 vq_cmd_w0.s.param1 = encr_data_len; 2310 2311 /* consider iv len */ 2312 encr_offset += iv_len; 2313 2314 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2315 outputlen = inputlen; 2316 2317 /* save space for offset ctrl & iv */ 2318 offset_vaddr = m_vaddr; 2319 offset_dma = m_dma; 2320 2321 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2322 m_dma += OFF_CTRL_LEN + iv_len; 2323 2324 /* DPTR has SG list */ 2325 in_buffer = m_vaddr; 2326 dptr_dma = m_dma; 2327 2328 ((uint16_t *)in_buffer)[0] = 0; 2329 ((uint16_t *)in_buffer)[1] = 0; 2330 2331 /* TODO Add error check if space will be sufficient */ 2332 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2333 2334 /* 2335 * Input Gather List 2336 */ 2337 i = 0; 2338 2339 /* Offset control word followed by iv */ 2340 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2341 if (unlikely((encr_offset >> 16))) { 2342 CPT_LOG_DP_ERR("Offset not supported"); 2343 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 2344 return; 2345 } 2346 2347 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2348 2349 /* IV */ 2350 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2351 params->iv_buf, iv_len); 2352 2353 /* Add input data */ 2354 size = inputlen - iv_len; 2355 if (size) { 2356 i = fill_sg_comp_from_iov(gather_comp, i, 2357 params->src_iov, 2358 0, &size, NULL, 0); 2359 if (unlikely(size)) { 2360 CPT_LOG_DP_ERR("Insufficient buffer space," 2361 " size %d needed", size); 2362 return; 2363 } 2364 } 2365 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2366 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2367 2368 /* 2369 * Output Scatter List 2370 */ 2371 2372 i = 0; 2373 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2374 2375 /* IV */ 2376 i = fill_sg_comp(scatter_comp, i, 2377 offset_dma + OFF_CTRL_LEN, 2378 iv_len); 2379 2380 /* Add output data */ 2381 size = outputlen - iv_len; 2382 if (size) { 2383 i = fill_sg_comp_from_iov(scatter_comp, i, 2384 params->dst_iov, 0, 2385 &size, NULL, 0); 2386 if (unlikely(size)) { 2387 CPT_LOG_DP_ERR("Insufficient buffer space," 2388 " size %d needed", size); 2389 return; 2390 } 2391 } 2392 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2393 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2394 2395 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2396 2397 /* This is DPTR len incase of SG mode */ 2398 vq_cmd_w0.s.dlen = size; 2399 2400 m_vaddr = (uint8_t *)m_vaddr + size; 2401 m_dma += size; 2402 2403 /* cpt alternate completion address saved earlier */ 2404 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2405 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2406 rptr_dma = c_dma - 8; 2407 2408 req->ist.ei1 = dptr_dma; 2409 req->ist.ei2 = rptr_dma; 2410 2411 /* 16 byte aligned cpt res address */ 2412 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2413 *req->completion_addr = COMPLETION_CODE_INIT; 2414 req->comp_baddr = c_dma; 2415 2416 /* Fill microcode part of instruction */ 2417 req->ist.ei0 = vq_cmd_w0.u64; 2418 2419 req->op = op; 2420 2421 *prep_req = req; 2422 return; 2423 } 2424 2425 static __rte_always_inline void * 2426 cpt_fc_dec_hmac_prep(uint32_t flags, 2427 uint64_t d_offs, 2428 uint64_t d_lens, 2429 fc_params_t *fc_params, 2430 void *op) 2431 { 2432 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2433 uint8_t fc_type; 2434 void *prep_req = NULL; 2435 2436 fc_type = ctx->fc_type; 2437 2438 if (likely(fc_type == FC_GEN)) { 2439 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2440 &prep_req); 2441 } else if (fc_type == ZUC_SNOW3G) { 2442 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2443 &prep_req); 2444 } else if (fc_type == KASUMI) { 2445 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2446 } 2447 2448 /* 2449 * For AUTH_ONLY case, 2450 * MC only supports digest generation and verification 2451 * should be done in software by memcmp() 2452 */ 2453 2454 return prep_req; 2455 } 2456 2457 static __rte_always_inline void *__rte_hot 2458 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2459 fc_params_t *fc_params, void *op) 2460 { 2461 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2462 uint8_t fc_type; 2463 void *prep_req = NULL; 2464 2465 fc_type = ctx->fc_type; 2466 2467 /* Common api for rest of the ops */ 2468 if (likely(fc_type == FC_GEN)) { 2469 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2470 &prep_req); 2471 } else if (fc_type == ZUC_SNOW3G) { 2472 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2473 &prep_req); 2474 } else if (fc_type == KASUMI) { 2475 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2476 &prep_req); 2477 } else if (fc_type == HASH_HMAC) { 2478 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2479 } 2480 2481 return prep_req; 2482 } 2483 2484 static __rte_always_inline int 2485 cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type, 2486 const uint8_t *key, uint16_t key_len, uint16_t mac_len) 2487 { 2488 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 2489 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 2490 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 2491 2492 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2493 uint32_t keyx[4]; 2494 2495 if (key_len != 16) 2496 return -1; 2497 /* No support for AEAD yet */ 2498 if (cpt_ctx->enc_cipher) 2499 return -1; 2500 /* For ZUC/SNOW3G/Kasumi */ 2501 switch (type) { 2502 case SNOW3G_UIA2: 2503 cpt_ctx->snow3g = 1; 2504 gen_key_snow3g(key, keyx); 2505 memcpy(zs_ctx->ci_key, keyx, key_len); 2506 cpt_ctx->fc_type = ZUC_SNOW3G; 2507 cpt_ctx->zsk_flags = 0x1; 2508 break; 2509 case ZUC_EIA3: 2510 cpt_ctx->snow3g = 0; 2511 memcpy(zs_ctx->ci_key, key, key_len); 2512 memcpy(zs_ctx->zuc_const, zuc_d, 32); 2513 cpt_ctx->fc_type = ZUC_SNOW3G; 2514 cpt_ctx->zsk_flags = 0x1; 2515 break; 2516 case KASUMI_F9_ECB: 2517 /* Kasumi ECB mode */ 2518 cpt_ctx->k_ecb = 1; 2519 memcpy(k_ctx->ci_key, key, key_len); 2520 cpt_ctx->fc_type = KASUMI; 2521 cpt_ctx->zsk_flags = 0x1; 2522 break; 2523 case KASUMI_F9_CBC: 2524 memcpy(k_ctx->ci_key, key, key_len); 2525 cpt_ctx->fc_type = KASUMI; 2526 cpt_ctx->zsk_flags = 0x1; 2527 break; 2528 default: 2529 return -1; 2530 } 2531 cpt_ctx->mac_len = 4; 2532 cpt_ctx->hash_type = type; 2533 return 0; 2534 } 2535 2536 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2537 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2538 cpt_ctx->fc_type = HASH_HMAC; 2539 } 2540 2541 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2542 return -1; 2543 2544 /* For GMAC auth, cipher must be NULL */ 2545 if (type == GMAC_TYPE) 2546 fctx->enc.enc_cipher = 0; 2547 2548 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2549 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2550 2551 if (key_len) { 2552 cpt_ctx->hmac = 1; 2553 2554 cpt_ctx->auth_key = rte_zmalloc(NULL, key_len, 8); 2555 if (cpt_ctx->auth_key == NULL) 2556 return -1; 2557 2558 cpt_ctx->auth_key_iova = rte_mem_virt2iova(cpt_ctx->auth_key); 2559 memcpy(cpt_ctx->auth_key, key, key_len); 2560 cpt_ctx->auth_key_len = key_len; 2561 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2562 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2563 2564 if (key_len <= 64) 2565 memcpy(fctx->hmac.opad, key, key_len); 2566 fctx->enc.auth_input_type = 1; 2567 } 2568 return 0; 2569 } 2570 2571 static __rte_always_inline int 2572 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2573 struct cpt_sess_misc *sess) 2574 { 2575 struct rte_crypto_aead_xform *aead_form; 2576 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2577 auth_type_t auth_type = 0; /* NULL Auth type */ 2578 uint32_t cipher_key_len = 0; 2579 uint8_t aes_gcm = 0; 2580 aead_form = &xform->aead; 2581 void *ctx = SESS_PRIV(sess); 2582 2583 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 2584 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2585 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2586 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) { 2587 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2588 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2589 } else { 2590 CPT_LOG_DP_ERR("Unknown aead operation\n"); 2591 return -1; 2592 } 2593 switch (aead_form->algo) { 2594 case RTE_CRYPTO_AEAD_AES_GCM: 2595 enc_type = AES_GCM; 2596 cipher_key_len = 16; 2597 aes_gcm = 1; 2598 break; 2599 case RTE_CRYPTO_AEAD_AES_CCM: 2600 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2601 aead_form->algo); 2602 return -1; 2603 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: 2604 enc_type = CHACHA20; 2605 auth_type = POLY1305; 2606 cipher_key_len = 32; 2607 sess->chacha_poly = 1; 2608 break; 2609 default: 2610 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2611 aead_form->algo); 2612 return -1; 2613 } 2614 if (aead_form->key.length < cipher_key_len) { 2615 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2616 (unsigned int long)aead_form->key.length); 2617 return -1; 2618 } 2619 sess->zsk_flag = 0; 2620 sess->aes_gcm = aes_gcm; 2621 sess->mac_len = aead_form->digest_length; 2622 sess->iv_offset = aead_form->iv.offset; 2623 sess->iv_length = aead_form->iv.length; 2624 sess->aad_length = aead_form->aad_length; 2625 2626 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2627 aead_form->key.length, NULL))) 2628 return -1; 2629 2630 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2631 aead_form->digest_length))) 2632 return -1; 2633 2634 return 0; 2635 } 2636 2637 static __rte_always_inline int 2638 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2639 struct cpt_sess_misc *sess) 2640 { 2641 struct rte_crypto_cipher_xform *c_form; 2642 struct cpt_ctx *ctx = SESS_PRIV(sess); 2643 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2644 uint32_t cipher_key_len = 0; 2645 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2646 2647 c_form = &xform->cipher; 2648 2649 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2650 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2651 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 2652 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2653 if (xform->next != NULL && 2654 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2655 /* Perform decryption followed by auth verify */ 2656 ctx->dec_auth = 1; 2657 } 2658 } else { 2659 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2660 return -1; 2661 } 2662 2663 switch (c_form->algo) { 2664 case RTE_CRYPTO_CIPHER_AES_CBC: 2665 enc_type = AES_CBC; 2666 cipher_key_len = 16; 2667 break; 2668 case RTE_CRYPTO_CIPHER_3DES_CBC: 2669 enc_type = DES3_CBC; 2670 cipher_key_len = 24; 2671 break; 2672 case RTE_CRYPTO_CIPHER_DES_CBC: 2673 /* DES is implemented using 3DES in hardware */ 2674 enc_type = DES3_CBC; 2675 cipher_key_len = 8; 2676 break; 2677 case RTE_CRYPTO_CIPHER_AES_CTR: 2678 enc_type = AES_CTR; 2679 cipher_key_len = 16; 2680 aes_ctr = 1; 2681 break; 2682 case RTE_CRYPTO_CIPHER_NULL: 2683 enc_type = 0; 2684 is_null = 1; 2685 break; 2686 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2687 enc_type = KASUMI_F8_ECB; 2688 cipher_key_len = 16; 2689 zsk_flag = K_F8; 2690 break; 2691 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2692 enc_type = SNOW3G_UEA2; 2693 cipher_key_len = 16; 2694 zsk_flag = ZS_EA; 2695 break; 2696 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2697 enc_type = ZUC_EEA3; 2698 cipher_key_len = 16; 2699 zsk_flag = ZS_EA; 2700 break; 2701 case RTE_CRYPTO_CIPHER_AES_XTS: 2702 enc_type = AES_XTS; 2703 cipher_key_len = 16; 2704 break; 2705 case RTE_CRYPTO_CIPHER_3DES_ECB: 2706 enc_type = DES3_ECB; 2707 cipher_key_len = 24; 2708 break; 2709 case RTE_CRYPTO_CIPHER_AES_ECB: 2710 enc_type = AES_ECB; 2711 cipher_key_len = 16; 2712 break; 2713 case RTE_CRYPTO_CIPHER_3DES_CTR: 2714 case RTE_CRYPTO_CIPHER_AES_F8: 2715 case RTE_CRYPTO_CIPHER_ARC4: 2716 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2717 c_form->algo); 2718 return -1; 2719 default: 2720 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2721 c_form->algo); 2722 return -1; 2723 } 2724 2725 if (c_form->key.length < cipher_key_len) { 2726 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2727 (unsigned long) c_form->key.length); 2728 return -1; 2729 } 2730 2731 sess->zsk_flag = zsk_flag; 2732 sess->aes_gcm = 0; 2733 sess->aes_ctr = aes_ctr; 2734 sess->iv_offset = c_form->iv.offset; 2735 sess->iv_length = c_form->iv.length; 2736 sess->is_null = is_null; 2737 2738 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2739 c_form->key.data, c_form->key.length, NULL))) 2740 return -1; 2741 2742 return 0; 2743 } 2744 2745 static __rte_always_inline int 2746 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2747 struct cpt_sess_misc *sess) 2748 { 2749 struct cpt_ctx *ctx = SESS_PRIV(sess); 2750 struct rte_crypto_auth_xform *a_form; 2751 auth_type_t auth_type = 0; /* NULL Auth type */ 2752 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2753 2754 if (xform->next != NULL && 2755 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2756 xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 2757 /* Perform auth followed by encryption */ 2758 ctx->auth_enc = 1; 2759 } 2760 2761 a_form = &xform->auth; 2762 2763 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2764 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2765 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2766 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2767 else { 2768 CPT_LOG_DP_ERR("Unknown auth operation"); 2769 return -1; 2770 } 2771 2772 switch (a_form->algo) { 2773 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2774 /* Fall through */ 2775 case RTE_CRYPTO_AUTH_SHA1: 2776 auth_type = SHA1_TYPE; 2777 break; 2778 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2779 case RTE_CRYPTO_AUTH_SHA256: 2780 auth_type = SHA2_SHA256; 2781 break; 2782 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2783 case RTE_CRYPTO_AUTH_SHA512: 2784 auth_type = SHA2_SHA512; 2785 break; 2786 case RTE_CRYPTO_AUTH_AES_GMAC: 2787 auth_type = GMAC_TYPE; 2788 aes_gcm = 1; 2789 break; 2790 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2791 case RTE_CRYPTO_AUTH_SHA224: 2792 auth_type = SHA2_SHA224; 2793 break; 2794 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2795 case RTE_CRYPTO_AUTH_SHA384: 2796 auth_type = SHA2_SHA384; 2797 break; 2798 case RTE_CRYPTO_AUTH_MD5_HMAC: 2799 case RTE_CRYPTO_AUTH_MD5: 2800 auth_type = MD5_TYPE; 2801 break; 2802 case RTE_CRYPTO_AUTH_KASUMI_F9: 2803 auth_type = KASUMI_F9_ECB; 2804 /* 2805 * Indicate that direction needs to be taken out 2806 * from end of src 2807 */ 2808 zsk_flag = K_F9; 2809 break; 2810 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2811 auth_type = SNOW3G_UIA2; 2812 zsk_flag = ZS_IA; 2813 break; 2814 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2815 auth_type = ZUC_EIA3; 2816 zsk_flag = ZS_IA; 2817 break; 2818 case RTE_CRYPTO_AUTH_NULL: 2819 auth_type = 0; 2820 is_null = 1; 2821 break; 2822 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2823 case RTE_CRYPTO_AUTH_AES_CMAC: 2824 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2825 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2826 a_form->algo); 2827 return -1; 2828 default: 2829 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2830 a_form->algo); 2831 return -1; 2832 } 2833 2834 sess->zsk_flag = zsk_flag; 2835 sess->aes_gcm = aes_gcm; 2836 sess->mac_len = a_form->digest_length; 2837 sess->is_null = is_null; 2838 if (zsk_flag) { 2839 sess->auth_iv_offset = a_form->iv.offset; 2840 sess->auth_iv_length = a_form->iv.length; 2841 } 2842 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2843 a_form->key.data, a_form->key.length, 2844 a_form->digest_length))) 2845 return -1; 2846 2847 return 0; 2848 } 2849 2850 static __rte_always_inline int 2851 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2852 struct cpt_sess_misc *sess) 2853 { 2854 struct rte_crypto_auth_xform *a_form; 2855 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2856 auth_type_t auth_type = 0; /* NULL Auth type */ 2857 void *ctx = SESS_PRIV(sess); 2858 2859 a_form = &xform->auth; 2860 2861 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2862 sess->cpt_op |= CPT_OP_ENCODE; 2863 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2864 sess->cpt_op |= CPT_OP_DECODE; 2865 else { 2866 CPT_LOG_DP_ERR("Unknown auth operation"); 2867 return -1; 2868 } 2869 2870 switch (a_form->algo) { 2871 case RTE_CRYPTO_AUTH_AES_GMAC: 2872 enc_type = AES_GCM; 2873 auth_type = GMAC_TYPE; 2874 break; 2875 default: 2876 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2877 a_form->algo); 2878 return -1; 2879 } 2880 2881 sess->zsk_flag = 0; 2882 sess->aes_gcm = 0; 2883 sess->is_gmac = 1; 2884 sess->iv_offset = a_form->iv.offset; 2885 sess->iv_length = a_form->iv.length; 2886 sess->mac_len = a_form->digest_length; 2887 2888 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2889 a_form->key.length, NULL))) 2890 return -1; 2891 2892 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2893 a_form->digest_length))) 2894 return -1; 2895 2896 return 0; 2897 } 2898 2899 static __rte_always_inline void * 2900 alloc_op_meta(struct rte_mbuf *m_src, 2901 buf_ptr_t *buf, 2902 int32_t len, 2903 struct rte_mempool *cpt_meta_pool) 2904 { 2905 uint8_t *mdata; 2906 2907 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2908 if (likely(m_src && (m_src->nb_segs == 1))) { 2909 int32_t tailroom; 2910 phys_addr_t mphys; 2911 2912 /* Check if tailroom is sufficient to hold meta data */ 2913 tailroom = rte_pktmbuf_tailroom(m_src); 2914 if (likely(tailroom > len + 8)) { 2915 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2916 mphys = m_src->buf_iova + m_src->buf_len; 2917 mdata -= len; 2918 mphys -= len; 2919 buf->vaddr = mdata; 2920 buf->dma_addr = mphys; 2921 buf->size = len; 2922 /* Indicate that this is a mbuf allocated mdata */ 2923 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2924 return mdata; 2925 } 2926 } 2927 #else 2928 RTE_SET_USED(m_src); 2929 #endif 2930 2931 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2932 return NULL; 2933 2934 buf->vaddr = mdata; 2935 buf->dma_addr = rte_mempool_virt2iova(mdata); 2936 buf->size = len; 2937 2938 return mdata; 2939 } 2940 2941 /** 2942 * cpt_free_metabuf - free metabuf to mempool. 2943 * @param instance: pointer to instance. 2944 * @param objp: pointer to the metabuf. 2945 */ 2946 static __rte_always_inline void 2947 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2948 { 2949 bool nofree = ((uintptr_t)mdata & 1ull); 2950 2951 if (likely(nofree)) 2952 return; 2953 rte_mempool_put(cpt_meta_pool, mdata); 2954 } 2955 2956 static __rte_always_inline uint32_t 2957 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2958 iov_ptr_t *iovec, uint32_t start_offset) 2959 { 2960 uint16_t index = 0; 2961 void *seg_data = NULL; 2962 phys_addr_t seg_phys; 2963 int32_t seg_size = 0; 2964 2965 if (!pkt) { 2966 iovec->buf_cnt = 0; 2967 return 0; 2968 } 2969 2970 if (!start_offset) { 2971 seg_data = rte_pktmbuf_mtod(pkt, void *); 2972 seg_phys = rte_pktmbuf_iova(pkt); 2973 seg_size = pkt->data_len; 2974 } else { 2975 while (start_offset >= pkt->data_len) { 2976 start_offset -= pkt->data_len; 2977 pkt = pkt->next; 2978 } 2979 2980 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2981 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset); 2982 seg_size = pkt->data_len - start_offset; 2983 if (!seg_size) 2984 return 1; 2985 } 2986 2987 /* first seg */ 2988 iovec->bufs[index].vaddr = seg_data; 2989 iovec->bufs[index].dma_addr = seg_phys; 2990 iovec->bufs[index].size = seg_size; 2991 index++; 2992 pkt = pkt->next; 2993 2994 while (unlikely(pkt != NULL)) { 2995 seg_data = rte_pktmbuf_mtod(pkt, void *); 2996 seg_phys = rte_pktmbuf_iova(pkt); 2997 seg_size = pkt->data_len; 2998 if (!seg_size) 2999 break; 3000 3001 iovec->bufs[index].vaddr = seg_data; 3002 iovec->bufs[index].dma_addr = seg_phys; 3003 iovec->bufs[index].size = seg_size; 3004 3005 index++; 3006 3007 pkt = pkt->next; 3008 } 3009 3010 iovec->buf_cnt = index; 3011 return 0; 3012 } 3013 3014 static __rte_always_inline uint32_t 3015 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 3016 fc_params_t *param, 3017 uint32_t *flags) 3018 { 3019 uint16_t index = 0; 3020 void *seg_data = NULL; 3021 phys_addr_t seg_phys; 3022 uint32_t seg_size = 0; 3023 iov_ptr_t *iovec; 3024 3025 seg_data = rte_pktmbuf_mtod(pkt, void *); 3026 seg_phys = rte_pktmbuf_iova(pkt); 3027 seg_size = pkt->data_len; 3028 3029 /* first seg */ 3030 if (likely(!pkt->next)) { 3031 uint32_t headroom, tailroom; 3032 3033 *flags |= SINGLE_BUF_INPLACE; 3034 headroom = rte_pktmbuf_headroom(pkt); 3035 tailroom = rte_pktmbuf_tailroom(pkt); 3036 if (likely((headroom >= 24) && 3037 (tailroom >= 8))) { 3038 /* In 83XX this is prerequivisit for Direct mode */ 3039 *flags |= SINGLE_BUF_HEADTAILROOM; 3040 } 3041 param->bufs[0].vaddr = seg_data; 3042 param->bufs[0].dma_addr = seg_phys; 3043 param->bufs[0].size = seg_size; 3044 return 0; 3045 } 3046 iovec = param->src_iov; 3047 iovec->bufs[index].vaddr = seg_data; 3048 iovec->bufs[index].dma_addr = seg_phys; 3049 iovec->bufs[index].size = seg_size; 3050 index++; 3051 pkt = pkt->next; 3052 3053 while (unlikely(pkt != NULL)) { 3054 seg_data = rte_pktmbuf_mtod(pkt, void *); 3055 seg_phys = rte_pktmbuf_iova(pkt); 3056 seg_size = pkt->data_len; 3057 3058 if (!seg_size) 3059 break; 3060 3061 iovec->bufs[index].vaddr = seg_data; 3062 iovec->bufs[index].dma_addr = seg_phys; 3063 iovec->bufs[index].size = seg_size; 3064 3065 index++; 3066 3067 pkt = pkt->next; 3068 } 3069 3070 iovec->buf_cnt = index; 3071 return 0; 3072 } 3073 3074 static __rte_always_inline int 3075 fill_fc_params(struct rte_crypto_op *cop, 3076 struct cpt_sess_misc *sess_misc, 3077 struct cpt_qp_meta_info *m_info, 3078 void **mdata_ptr, 3079 void **prep_req) 3080 { 3081 uint32_t space = 0; 3082 struct rte_crypto_sym_op *sym_op = cop->sym; 3083 struct cpt_ctx *ctx = SESS_PRIV(sess_misc); 3084 void *mdata = NULL; 3085 uintptr_t *op; 3086 uint32_t mc_hash_off; 3087 uint32_t flags = 0; 3088 uint64_t d_offs, d_lens; 3089 struct rte_mbuf *m_src, *m_dst; 3090 uint8_t cpt_op = sess_misc->cpt_op; 3091 #ifdef CPT_ALWAYS_USE_SG_MODE 3092 uint8_t inplace = 0; 3093 #else 3094 uint8_t inplace = 1; 3095 #endif 3096 fc_params_t fc_params; 3097 char src[SRC_IOV_SIZE]; 3098 char dst[SRC_IOV_SIZE]; 3099 uint32_t iv_buf[4]; 3100 int ret; 3101 3102 if (likely(sess_misc->iv_length)) { 3103 flags |= VALID_IV_BUF; 3104 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3105 uint8_t *, sess_misc->iv_offset); 3106 if (sess_misc->aes_ctr && 3107 unlikely(sess_misc->iv_length != 16)) { 3108 memcpy((uint8_t *)iv_buf, 3109 rte_crypto_op_ctod_offset(cop, 3110 uint8_t *, sess_misc->iv_offset), 12); 3111 iv_buf[3] = rte_cpu_to_be_32(0x1); 3112 fc_params.iv_buf = iv_buf; 3113 } 3114 } 3115 3116 if (sess_misc->zsk_flag) { 3117 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3118 uint8_t *, 3119 sess_misc->auth_iv_offset); 3120 if (sess_misc->zsk_flag != ZS_EA) 3121 inplace = 0; 3122 } 3123 m_src = sym_op->m_src; 3124 m_dst = sym_op->m_dst; 3125 3126 if (sess_misc->aes_gcm || sess_misc->chacha_poly) { 3127 uint8_t *salt; 3128 uint8_t *aad_data; 3129 uint16_t aad_len; 3130 3131 d_offs = sym_op->aead.data.offset; 3132 d_lens = sym_op->aead.data.length; 3133 mc_hash_off = sym_op->aead.data.offset + 3134 sym_op->aead.data.length; 3135 3136 aad_data = sym_op->aead.aad.data; 3137 aad_len = sess_misc->aad_length; 3138 if (likely((aad_data + aad_len) == 3139 rte_pktmbuf_mtod_offset(m_src, 3140 uint8_t *, 3141 sym_op->aead.data.offset))) { 3142 d_offs = (d_offs - aad_len) | (d_offs << 16); 3143 d_lens = (d_lens + aad_len) | (d_lens << 32); 3144 } else { 3145 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3146 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3147 fc_params.aad_buf.size = aad_len; 3148 flags |= VALID_AAD_BUF; 3149 inplace = 0; 3150 d_offs = d_offs << 16; 3151 d_lens = d_lens << 32; 3152 } 3153 3154 salt = fc_params.iv_buf; 3155 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3156 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3157 sess_misc->salt = *(uint32_t *)salt; 3158 } 3159 fc_params.iv_buf = salt + 4; 3160 if (likely(sess_misc->mac_len)) { 3161 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3162 m_src; 3163 3164 if (!m) 3165 m = m_src; 3166 3167 /* hmac immediately following data is best case */ 3168 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3169 mc_hash_off != 3170 (uint8_t *)sym_op->aead.digest.data)) { 3171 flags |= VALID_MAC_BUF; 3172 fc_params.mac_buf.size = sess_misc->mac_len; 3173 fc_params.mac_buf.vaddr = 3174 sym_op->aead.digest.data; 3175 fc_params.mac_buf.dma_addr = 3176 sym_op->aead.digest.phys_addr; 3177 inplace = 0; 3178 } 3179 } 3180 } else { 3181 d_offs = sym_op->cipher.data.offset; 3182 d_lens = sym_op->cipher.data.length; 3183 mc_hash_off = sym_op->cipher.data.offset + 3184 sym_op->cipher.data.length; 3185 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3186 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3187 3188 if (mc_hash_off < (sym_op->auth.data.offset + 3189 sym_op->auth.data.length)){ 3190 mc_hash_off = (sym_op->auth.data.offset + 3191 sym_op->auth.data.length); 3192 } 3193 /* for gmac, salt should be updated like in gcm */ 3194 if (unlikely(sess_misc->is_gmac)) { 3195 uint8_t *salt; 3196 salt = fc_params.iv_buf; 3197 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3198 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3199 sess_misc->salt = *(uint32_t *)salt; 3200 } 3201 fc_params.iv_buf = salt + 4; 3202 } 3203 if (likely(sess_misc->mac_len)) { 3204 struct rte_mbuf *m; 3205 3206 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3207 if (!m) 3208 m = m_src; 3209 3210 /* hmac immediately following data is best case */ 3211 if (!ctx->dec_auth && !ctx->auth_enc && 3212 (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3213 mc_hash_off != 3214 (uint8_t *)sym_op->auth.digest.data))) { 3215 flags |= VALID_MAC_BUF; 3216 fc_params.mac_buf.size = 3217 sess_misc->mac_len; 3218 fc_params.mac_buf.vaddr = 3219 sym_op->auth.digest.data; 3220 fc_params.mac_buf.dma_addr = 3221 sym_op->auth.digest.phys_addr; 3222 inplace = 0; 3223 } 3224 } 3225 } 3226 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3227 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3228 3229 if (!ctx->dec_auth && 3230 unlikely(sess_misc->is_null || 3231 sess_misc->cpt_op == CPT_OP_DECODE)) 3232 inplace = 0; 3233 3234 if (likely(!m_dst && inplace)) { 3235 /* Case of single buffer without AAD buf or 3236 * separate mac buf in place and 3237 * not air crypto 3238 */ 3239 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3240 3241 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3242 &fc_params, 3243 &flags))) { 3244 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3245 ret = -EINVAL; 3246 goto err_exit; 3247 } 3248 3249 } else { 3250 /* Out of place processing */ 3251 fc_params.src_iov = (void *)src; 3252 fc_params.dst_iov = (void *)dst; 3253 3254 /* Store SG I/O in the api for reuse */ 3255 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3256 CPT_LOG_DP_ERR("Prepare src iov failed"); 3257 ret = -EINVAL; 3258 goto err_exit; 3259 } 3260 3261 if (unlikely(m_dst != NULL)) { 3262 uint32_t pkt_len; 3263 3264 /* Try to make room as much as src has */ 3265 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3266 3267 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3268 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3269 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3270 CPT_LOG_DP_ERR("Not enough space in " 3271 "m_dst %p, need %u" 3272 " more", 3273 m_dst, pkt_len); 3274 ret = -EINVAL; 3275 goto err_exit; 3276 } 3277 } 3278 3279 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3280 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3281 "m_dst %p", m_dst); 3282 ret = -EINVAL; 3283 goto err_exit; 3284 } 3285 } else { 3286 fc_params.dst_iov = (void *)src; 3287 } 3288 } 3289 3290 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3291 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3292 m_info->lb_mlen, m_info->pool); 3293 else 3294 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3295 m_info->sg_mlen, m_info->pool); 3296 3297 if (unlikely(mdata == NULL)) { 3298 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3299 ret = -ENOMEM; 3300 goto err_exit; 3301 } 3302 3303 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3304 op[0] = (uintptr_t)mdata; 3305 op[1] = (uintptr_t)cop; 3306 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3307 space += 4 * sizeof(uint64_t); 3308 3309 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3310 fc_params.meta_buf.dma_addr += space; 3311 fc_params.meta_buf.size -= space; 3312 3313 /* Finally prepare the instruction */ 3314 if (cpt_op & CPT_OP_ENCODE) 3315 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3316 &fc_params, op); 3317 else 3318 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3319 &fc_params, op); 3320 3321 if (unlikely(*prep_req == NULL)) { 3322 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3323 ret = -EINVAL; 3324 goto free_mdata_and_exit; 3325 } 3326 3327 *mdata_ptr = mdata; 3328 3329 return 0; 3330 3331 free_mdata_and_exit: 3332 free_op_meta(mdata, m_info->pool); 3333 err_exit: 3334 return ret; 3335 } 3336 3337 static __rte_always_inline void 3338 compl_auth_verify(struct rte_crypto_op *op, 3339 uint8_t *gen_mac, 3340 uint64_t mac_len) 3341 { 3342 uint8_t *mac; 3343 struct rte_crypto_sym_op *sym_op = op->sym; 3344 3345 if (sym_op->auth.digest.data) 3346 mac = sym_op->auth.digest.data; 3347 else 3348 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3349 uint8_t *, 3350 sym_op->auth.data.length + 3351 sym_op->auth.data.offset); 3352 if (!mac) { 3353 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3354 return; 3355 } 3356 3357 if (memcmp(mac, gen_mac, mac_len)) 3358 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3359 else 3360 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3361 } 3362 3363 static __rte_always_inline void 3364 find_kasumif9_direction_and_length(uint8_t *src, 3365 uint32_t counter_num_bytes, 3366 uint32_t *addr_length_in_bits, 3367 uint8_t *addr_direction) 3368 { 3369 uint8_t found = 0; 3370 uint32_t pos; 3371 uint8_t last_byte; 3372 while (!found && counter_num_bytes > 0) { 3373 counter_num_bytes--; 3374 if (src[counter_num_bytes] == 0x00) 3375 continue; 3376 pos = rte_bsf32(src[counter_num_bytes]); 3377 if (pos == 7) { 3378 if (likely(counter_num_bytes > 0)) { 3379 last_byte = src[counter_num_bytes - 1]; 3380 *addr_direction = last_byte & 0x1; 3381 *addr_length_in_bits = counter_num_bytes * 8 3382 - 1; 3383 } 3384 } else { 3385 last_byte = src[counter_num_bytes]; 3386 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3387 *addr_length_in_bits = counter_num_bytes * 8 3388 + (8 - (pos + 2)); 3389 } 3390 found = 1; 3391 } 3392 } 3393 3394 /* 3395 * This handles all auth only except AES_GMAC 3396 */ 3397 static __rte_always_inline int 3398 fill_digest_params(struct rte_crypto_op *cop, 3399 struct cpt_sess_misc *sess, 3400 struct cpt_qp_meta_info *m_info, 3401 void **mdata_ptr, 3402 void **prep_req) 3403 { 3404 uint32_t space = 0; 3405 struct rte_crypto_sym_op *sym_op = cop->sym; 3406 void *mdata; 3407 phys_addr_t mphys; 3408 uint64_t *op; 3409 uint32_t auth_range_off; 3410 uint32_t flags = 0; 3411 uint64_t d_offs = 0, d_lens; 3412 struct rte_mbuf *m_src, *m_dst; 3413 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3414 uint16_t mac_len = sess->mac_len; 3415 fc_params_t params; 3416 char src[SRC_IOV_SIZE]; 3417 uint8_t iv_buf[16]; 3418 int ret; 3419 3420 memset(¶ms, 0, sizeof(fc_params_t)); 3421 3422 m_src = sym_op->m_src; 3423 3424 /* For just digest lets force mempool alloc */ 3425 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3426 m_info->pool); 3427 if (mdata == NULL) { 3428 ret = -ENOMEM; 3429 goto err_exit; 3430 } 3431 3432 mphys = params.meta_buf.dma_addr; 3433 3434 op = mdata; 3435 op[0] = (uintptr_t)mdata; 3436 op[1] = (uintptr_t)cop; 3437 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3438 space += 4 * sizeof(uint64_t); 3439 3440 auth_range_off = sym_op->auth.data.offset; 3441 3442 flags = VALID_MAC_BUF; 3443 params.src_iov = (void *)src; 3444 if (unlikely(sess->zsk_flag)) { 3445 /* 3446 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3447 * we will send pass through even for auth only case, 3448 * let MC handle it 3449 */ 3450 d_offs = auth_range_off; 3451 auth_range_off = 0; 3452 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3453 uint8_t *, sess->auth_iv_offset); 3454 if (sess->zsk_flag == K_F9) { 3455 uint32_t length_in_bits, num_bytes; 3456 uint8_t *src, direction = 0; 3457 3458 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3459 uint8_t *), 8); 3460 /* 3461 * This is kasumi f9, take direction from 3462 * source buffer 3463 */ 3464 length_in_bits = cop->sym->auth.data.length; 3465 num_bytes = (length_in_bits >> 3); 3466 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3467 find_kasumif9_direction_and_length(src, 3468 num_bytes, 3469 &length_in_bits, 3470 &direction); 3471 length_in_bits -= 64; 3472 cop->sym->auth.data.offset += 64; 3473 d_offs = cop->sym->auth.data.offset; 3474 auth_range_off = d_offs / 8; 3475 cop->sym->auth.data.length = length_in_bits; 3476 3477 /* Store it at end of auth iv */ 3478 iv_buf[8] = direction; 3479 params.auth_iv_buf = iv_buf; 3480 } 3481 } 3482 3483 d_lens = sym_op->auth.data.length; 3484 3485 params.ctx_buf.vaddr = SESS_PRIV(sess); 3486 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3487 3488 if (auth_op == CPT_OP_AUTH_GENERATE) { 3489 if (sym_op->auth.digest.data) { 3490 /* 3491 * Digest to be generated 3492 * in separate buffer 3493 */ 3494 params.mac_buf.size = 3495 sess->mac_len; 3496 params.mac_buf.vaddr = 3497 sym_op->auth.digest.data; 3498 params.mac_buf.dma_addr = 3499 sym_op->auth.digest.phys_addr; 3500 } else { 3501 uint32_t off = sym_op->auth.data.offset + 3502 sym_op->auth.data.length; 3503 int32_t dlen, space; 3504 3505 m_dst = sym_op->m_dst ? 3506 sym_op->m_dst : sym_op->m_src; 3507 dlen = rte_pktmbuf_pkt_len(m_dst); 3508 3509 space = off + mac_len - dlen; 3510 if (space > 0) 3511 if (!rte_pktmbuf_append(m_dst, space)) { 3512 CPT_LOG_DP_ERR("Failed to extend " 3513 "mbuf by %uB", space); 3514 ret = -EINVAL; 3515 goto free_mdata_and_exit; 3516 } 3517 3518 params.mac_buf.vaddr = 3519 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3520 params.mac_buf.dma_addr = 3521 rte_pktmbuf_iova_offset(m_dst, off); 3522 params.mac_buf.size = mac_len; 3523 } 3524 } else { 3525 /* Need space for storing generated mac */ 3526 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3527 params.mac_buf.dma_addr = mphys + space; 3528 params.mac_buf.size = mac_len; 3529 space += RTE_ALIGN_CEIL(mac_len, 8); 3530 op[2] = (uintptr_t)params.mac_buf.vaddr; 3531 op[3] = mac_len; 3532 } 3533 3534 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3535 params.meta_buf.dma_addr = mphys + space; 3536 params.meta_buf.size -= space; 3537 3538 /* Out of place processing */ 3539 params.src_iov = (void *)src; 3540 3541 /*Store SG I/O in the api for reuse */ 3542 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3543 CPT_LOG_DP_ERR("Prepare src iov failed"); 3544 ret = -EINVAL; 3545 goto free_mdata_and_exit; 3546 } 3547 3548 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3549 if (unlikely(*prep_req == NULL)) { 3550 ret = -EINVAL; 3551 goto free_mdata_and_exit; 3552 } 3553 3554 *mdata_ptr = mdata; 3555 3556 return 0; 3557 3558 free_mdata_and_exit: 3559 free_op_meta(mdata, m_info->pool); 3560 err_exit: 3561 return ret; 3562 } 3563 3564 #endif /*_CPT_UCODE_H_ */ 3565