1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline void 26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 27 { 28 int i, base; 29 30 for (i = 0; i < 4; i++) { 31 base = 4 * i; 32 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 33 (ck[base + 2] << 8) | (ck[base + 3]); 34 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 35 } 36 } 37 38 static __rte_always_inline int 39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) 40 { 41 uint16_t mac_len = auth->digest_length; 42 int ret; 43 44 switch (auth->algo) { 45 case RTE_CRYPTO_AUTH_MD5: 46 case RTE_CRYPTO_AUTH_MD5_HMAC: 47 ret = (mac_len == 16) ? 0 : -1; 48 break; 49 case RTE_CRYPTO_AUTH_SHA1: 50 case RTE_CRYPTO_AUTH_SHA1_HMAC: 51 ret = (mac_len == 20) ? 0 : -1; 52 break; 53 case RTE_CRYPTO_AUTH_SHA224: 54 case RTE_CRYPTO_AUTH_SHA224_HMAC: 55 ret = (mac_len == 28) ? 0 : -1; 56 break; 57 case RTE_CRYPTO_AUTH_SHA256: 58 case RTE_CRYPTO_AUTH_SHA256_HMAC: 59 ret = (mac_len == 32) ? 0 : -1; 60 break; 61 case RTE_CRYPTO_AUTH_SHA384: 62 case RTE_CRYPTO_AUTH_SHA384_HMAC: 63 ret = (mac_len == 48) ? 0 : -1; 64 break; 65 case RTE_CRYPTO_AUTH_SHA512: 66 case RTE_CRYPTO_AUTH_SHA512_HMAC: 67 ret = (mac_len == 64) ? 0 : -1; 68 break; 69 case RTE_CRYPTO_AUTH_NULL: 70 ret = 0; 71 break; 72 default: 73 ret = -1; 74 } 75 76 return ret; 77 } 78 79 static __rte_always_inline void 80 cpt_fc_salt_update(struct cpt_ctx *cpt_ctx, 81 uint8_t *salt) 82 { 83 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 84 memcpy(fctx->enc.encr_iv, salt, 4); 85 } 86 87 static __rte_always_inline int 88 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 89 { 90 switch (key_len) { 91 case 16: 92 case 24: 93 case 32: 94 return 0; 95 default: 96 return -1; 97 } 98 } 99 100 static __rte_always_inline int 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 102 { 103 int fc_type = 0; 104 switch (type) { 105 case PASSTHROUGH: 106 fc_type = FC_GEN; 107 break; 108 case DES3_CBC: 109 case DES3_ECB: 110 fc_type = FC_GEN; 111 break; 112 case AES_CBC: 113 case AES_ECB: 114 case AES_CFB: 115 case AES_CTR: 116 case AES_GCM: 117 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 118 return -1; 119 fc_type = FC_GEN; 120 break; 121 case CHACHA20: 122 fc_type = FC_GEN; 123 break; 124 case AES_XTS: 125 key_len = key_len / 2; 126 if (unlikely(key_len == 24)) { 127 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 128 return -1; 129 } 130 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 131 return -1; 132 fc_type = FC_GEN; 133 break; 134 case ZUC_EEA3: 135 case SNOW3G_UEA2: 136 if (unlikely(key_len != 16)) 137 return -1; 138 /* No support for AEAD yet */ 139 if (unlikely(ctx->hash_type)) 140 return -1; 141 fc_type = ZUC_SNOW3G; 142 break; 143 case KASUMI_F8_CBC: 144 case KASUMI_F8_ECB: 145 if (unlikely(key_len != 16)) 146 return -1; 147 /* No support for AEAD yet */ 148 if (unlikely(ctx->hash_type)) 149 return -1; 150 fc_type = KASUMI; 151 break; 152 default: 153 return -1; 154 } 155 156 ctx->fc_type = fc_type; 157 return 0; 158 } 159 160 static __rte_always_inline void 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 162 { 163 cpt_ctx->enc_cipher = 0; 164 fctx->enc.enc_cipher = 0; 165 } 166 167 static __rte_always_inline void 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 169 { 170 mc_aes_type_t aes_key_type = 0; 171 switch (key_len) { 172 case 16: 173 aes_key_type = AES_128_BIT; 174 break; 175 case 24: 176 aes_key_type = AES_192_BIT; 177 break; 178 case 32: 179 aes_key_type = AES_256_BIT; 180 break; 181 default: 182 /* This should not happen */ 183 CPT_LOG_DP_ERR("Invalid AES key len"); 184 return; 185 } 186 fctx->enc.aes_key = aes_key_type; 187 } 188 189 static __rte_always_inline void 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 191 uint16_t key_len) 192 { 193 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 194 uint32_t keyx[4]; 195 196 cpt_ctx->snow3g = 1; 197 gen_key_snow3g(key, keyx); 198 memcpy(zs_ctx->ci_key, keyx, key_len); 199 cpt_ctx->zsk_flags = 0; 200 } 201 202 static __rte_always_inline void 203 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 204 uint16_t key_len) 205 { 206 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 207 208 cpt_ctx->snow3g = 0; 209 memcpy(zs_ctx->ci_key, key, key_len); 210 memcpy(zs_ctx->zuc_const, zuc_d, 32); 211 cpt_ctx->zsk_flags = 0; 212 } 213 214 static __rte_always_inline void 215 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 216 uint16_t key_len) 217 { 218 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 219 220 cpt_ctx->k_ecb = 1; 221 memcpy(k_ctx->ci_key, key, key_len); 222 cpt_ctx->zsk_flags = 0; 223 } 224 225 static __rte_always_inline void 226 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 227 uint16_t key_len) 228 { 229 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 230 231 memcpy(k_ctx->ci_key, key, key_len); 232 cpt_ctx->zsk_flags = 0; 233 } 234 235 static __rte_always_inline int 236 cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type, 237 const uint8_t *key, uint16_t key_len, uint8_t *salt) 238 { 239 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 240 int ret; 241 242 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 243 if (unlikely(ret)) 244 return -1; 245 246 if (cpt_ctx->fc_type == FC_GEN) { 247 /* 248 * We need to always say IV is from DPTR as user can 249 * sometimes iverride IV per operation. 250 */ 251 fctx->enc.iv_source = CPT_FROM_DPTR; 252 253 if (cpt_ctx->auth_key_len > 64) 254 return -1; 255 } 256 257 switch (type) { 258 case PASSTHROUGH: 259 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 260 goto success; 261 case DES3_CBC: 262 /* CPT performs DES using 3DES with the 8B DES-key 263 * replicated 2 more times to match the 24B 3DES-key. 264 * Eg. If org. key is "0x0a 0x0b", then new key is 265 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 266 */ 267 if (key_len == 8) { 268 /* Skipping the first 8B as it will be copied 269 * in the regular code flow 270 */ 271 memcpy(fctx->enc.encr_key+key_len, key, key_len); 272 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 273 } 274 break; 275 case DES3_ECB: 276 /* For DES3_ECB IV need to be from CTX. */ 277 fctx->enc.iv_source = CPT_FROM_CTX; 278 break; 279 case AES_CBC: 280 case AES_ECB: 281 case AES_CFB: 282 case AES_CTR: 283 case CHACHA20: 284 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 285 break; 286 case AES_GCM: 287 /* Even though iv source is from dptr, 288 * aes_gcm salt is taken from ctx 289 */ 290 if (salt) { 291 memcpy(fctx->enc.encr_iv, salt, 4); 292 /* Assuming it was just salt update 293 * and nothing else 294 */ 295 if (!key) 296 goto success; 297 } 298 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 299 break; 300 case AES_XTS: 301 key_len = key_len / 2; 302 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 303 304 /* Copy key2 for XTS into ipad */ 305 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 306 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 307 break; 308 case SNOW3G_UEA2: 309 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 310 goto success; 311 case ZUC_EEA3: 312 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 313 goto success; 314 case KASUMI_F8_ECB: 315 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 316 goto success; 317 case KASUMI_F8_CBC: 318 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 319 goto success; 320 default: 321 return -1; 322 } 323 324 /* Only for FC_GEN case */ 325 326 /* For GMAC auth, cipher must be NULL */ 327 if (cpt_ctx->hash_type != GMAC_TYPE) 328 fctx->enc.enc_cipher = type; 329 330 memcpy(fctx->enc.encr_key, key, key_len); 331 332 success: 333 cpt_ctx->enc_cipher = type; 334 335 return 0; 336 } 337 338 static __rte_always_inline uint32_t 339 fill_sg_comp(sg_comp_t *list, 340 uint32_t i, 341 phys_addr_t dma_addr, 342 uint32_t size) 343 { 344 sg_comp_t *to = &list[i>>2]; 345 346 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 347 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 348 i++; 349 return i; 350 } 351 352 static __rte_always_inline uint32_t 353 fill_sg_comp_from_buf(sg_comp_t *list, 354 uint32_t i, 355 buf_ptr_t *from) 356 { 357 sg_comp_t *to = &list[i>>2]; 358 359 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 360 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 361 i++; 362 return i; 363 } 364 365 static __rte_always_inline uint32_t 366 fill_sg_comp_from_buf_min(sg_comp_t *list, 367 uint32_t i, 368 buf_ptr_t *from, 369 uint32_t *psize) 370 { 371 sg_comp_t *to = &list[i >> 2]; 372 uint32_t size = *psize; 373 uint32_t e_len; 374 375 e_len = (size > from->size) ? from->size : size; 376 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 377 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 378 *psize -= e_len; 379 i++; 380 return i; 381 } 382 383 /* 384 * This fills the MC expected SGIO list 385 * from IOV given by user. 386 */ 387 static __rte_always_inline uint32_t 388 fill_sg_comp_from_iov(sg_comp_t *list, 389 uint32_t i, 390 iov_ptr_t *from, uint32_t from_offset, 391 uint32_t *psize, buf_ptr_t *extra_buf, 392 uint32_t extra_offset) 393 { 394 int32_t j; 395 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 396 uint32_t size = *psize; 397 buf_ptr_t *bufs; 398 399 bufs = from->bufs; 400 for (j = 0; (j < from->buf_cnt) && size; j++) { 401 phys_addr_t e_dma_addr; 402 uint32_t e_len; 403 sg_comp_t *to = &list[i >> 2]; 404 405 if (unlikely(from_offset)) { 406 if (from_offset >= bufs[j].size) { 407 from_offset -= bufs[j].size; 408 continue; 409 } 410 e_dma_addr = bufs[j].dma_addr + from_offset; 411 e_len = (size > (bufs[j].size - from_offset)) ? 412 (bufs[j].size - from_offset) : size; 413 from_offset = 0; 414 } else { 415 e_dma_addr = bufs[j].dma_addr; 416 e_len = (size > bufs[j].size) ? 417 bufs[j].size : size; 418 } 419 420 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 421 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 422 423 if (extra_len && (e_len >= extra_offset)) { 424 /* Break the data at given offset */ 425 uint32_t next_len = e_len - extra_offset; 426 phys_addr_t next_dma = e_dma_addr + extra_offset; 427 428 if (!extra_offset) { 429 i--; 430 } else { 431 e_len = extra_offset; 432 size -= e_len; 433 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 434 } 435 436 extra_len = RTE_MIN(extra_len, size); 437 /* Insert extra data ptr */ 438 if (extra_len) { 439 i++; 440 to = &list[i >> 2]; 441 to->u.s.len[i % 4] = 442 rte_cpu_to_be_16(extra_len); 443 to->ptr[i % 4] = 444 rte_cpu_to_be_64(extra_buf->dma_addr); 445 size -= extra_len; 446 } 447 448 next_len = RTE_MIN(next_len, size); 449 /* insert the rest of the data */ 450 if (next_len) { 451 i++; 452 to = &list[i >> 2]; 453 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 454 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 455 size -= next_len; 456 } 457 extra_len = 0; 458 459 } else { 460 size -= e_len; 461 } 462 if (extra_offset) 463 extra_offset -= size; 464 i++; 465 } 466 467 *psize = size; 468 return (uint32_t)i; 469 } 470 471 static __rte_always_inline void 472 cpt_digest_gen_prep(uint32_t flags, 473 uint64_t d_lens, 474 digest_params_t *params, 475 void *op, 476 void **prep_req) 477 { 478 struct cpt_request_info *req; 479 uint32_t size, i; 480 uint16_t data_len, mac_len, key_len; 481 auth_type_t hash_type; 482 buf_ptr_t *meta_p; 483 struct cpt_ctx *ctx; 484 sg_comp_t *gather_comp; 485 sg_comp_t *scatter_comp; 486 uint8_t *in_buffer; 487 uint32_t g_size_bytes, s_size_bytes; 488 uint64_t dptr_dma, rptr_dma; 489 vq_cmd_word0_t vq_cmd_w0; 490 void *c_vaddr, *m_vaddr; 491 uint64_t c_dma, m_dma; 492 493 ctx = params->ctx_buf.vaddr; 494 meta_p = ¶ms->meta_buf; 495 496 m_vaddr = meta_p->vaddr; 497 m_dma = meta_p->dma_addr; 498 499 /* 500 * Save initial space that followed app data for completion code & 501 * alternate completion code to fall in same cache line as app data 502 */ 503 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 504 m_dma += COMPLETION_CODE_SIZE; 505 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 506 (uint8_t *)m_vaddr; 507 c_vaddr = (uint8_t *)m_vaddr + size; 508 c_dma = m_dma + size; 509 size += sizeof(cpt_res_s_t); 510 511 m_vaddr = (uint8_t *)m_vaddr + size; 512 m_dma += size; 513 514 req = m_vaddr; 515 516 size = sizeof(struct cpt_request_info); 517 m_vaddr = (uint8_t *)m_vaddr + size; 518 m_dma += size; 519 520 hash_type = ctx->hash_type; 521 mac_len = ctx->mac_len; 522 key_len = ctx->auth_key_len; 523 data_len = AUTH_DLEN(d_lens); 524 525 /*GP op header */ 526 vq_cmd_w0.s.opcode.minor = 0; 527 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 528 if (ctx->hmac) { 529 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 530 vq_cmd_w0.s.param1 = key_len; 531 vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8); 532 } else { 533 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 534 vq_cmd_w0.s.param1 = 0; 535 vq_cmd_w0.s.dlen = data_len; 536 } 537 538 /* Null auth only case enters the if */ 539 if (unlikely(!hash_type && !ctx->enc_cipher)) { 540 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC; 541 /* Minor op is passthrough */ 542 vq_cmd_w0.s.opcode.minor = 0x03; 543 /* Send out completion code only */ 544 vq_cmd_w0.s.param2 = 0x1; 545 } 546 547 /* DPTR has SG list */ 548 in_buffer = m_vaddr; 549 dptr_dma = m_dma; 550 551 ((uint16_t *)in_buffer)[0] = 0; 552 ((uint16_t *)in_buffer)[1] = 0; 553 554 /* TODO Add error check if space will be sufficient */ 555 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 556 557 /* 558 * Input gather list 559 */ 560 561 i = 0; 562 563 if (ctx->hmac) { 564 uint64_t k_dma = params->ctx_buf.dma_addr + 565 offsetof(struct cpt_ctx, auth_key); 566 /* Key */ 567 i = fill_sg_comp(gather_comp, i, k_dma, 568 RTE_ALIGN_CEIL(key_len, 8)); 569 } 570 571 /* input data */ 572 size = data_len; 573 if (size) { 574 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 575 0, &size, NULL, 0); 576 if (unlikely(size)) { 577 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 578 " by %dB", size); 579 return; 580 } 581 } else { 582 /* 583 * Looks like we need to support zero data 584 * gather ptr in case of hash & hmac 585 */ 586 i++; 587 } 588 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 589 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 590 591 /* 592 * Output Gather list 593 */ 594 595 i = 0; 596 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 597 598 if (flags & VALID_MAC_BUF) { 599 if (unlikely(params->mac_buf.size < mac_len)) { 600 CPT_LOG_DP_ERR("Insufficient MAC size"); 601 return; 602 } 603 604 size = mac_len; 605 i = fill_sg_comp_from_buf_min(scatter_comp, i, 606 ¶ms->mac_buf, &size); 607 } else { 608 size = mac_len; 609 i = fill_sg_comp_from_iov(scatter_comp, i, 610 params->src_iov, data_len, 611 &size, NULL, 0); 612 if (unlikely(size)) { 613 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 614 " %dB", size); 615 return; 616 } 617 } 618 619 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 620 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 621 622 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 623 624 /* This is DPTR len incase of SG mode */ 625 vq_cmd_w0.s.dlen = size; 626 627 m_vaddr = (uint8_t *)m_vaddr + size; 628 m_dma += size; 629 630 /* cpt alternate completion address saved earlier */ 631 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 632 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 633 rptr_dma = c_dma - 8; 634 635 req->ist.ei1 = dptr_dma; 636 req->ist.ei2 = rptr_dma; 637 638 /* 16 byte aligned cpt res address */ 639 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 640 *req->completion_addr = COMPLETION_CODE_INIT; 641 req->comp_baddr = c_dma; 642 643 /* Fill microcode part of instruction */ 644 req->ist.ei0 = vq_cmd_w0.u64; 645 646 req->op = op; 647 648 *prep_req = req; 649 return; 650 } 651 652 static __rte_always_inline void 653 cpt_enc_hmac_prep(uint32_t flags, 654 uint64_t d_offs, 655 uint64_t d_lens, 656 fc_params_t *fc_params, 657 void *op, 658 void **prep_req) 659 { 660 uint32_t iv_offset = 0; 661 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 662 struct cpt_ctx *cpt_ctx; 663 uint32_t cipher_type, hash_type; 664 uint32_t mac_len, size; 665 uint8_t iv_len = 16; 666 struct cpt_request_info *req; 667 buf_ptr_t *meta_p, *aad_buf = NULL; 668 uint32_t encr_offset, auth_offset; 669 uint32_t encr_data_len, auth_data_len, aad_len = 0; 670 uint32_t passthrough_len = 0; 671 void *m_vaddr, *offset_vaddr; 672 uint64_t m_dma, offset_dma; 673 vq_cmd_word0_t vq_cmd_w0; 674 void *c_vaddr; 675 uint64_t c_dma; 676 677 meta_p = &fc_params->meta_buf; 678 m_vaddr = meta_p->vaddr; 679 m_dma = meta_p->dma_addr; 680 681 encr_offset = ENCR_OFFSET(d_offs); 682 auth_offset = AUTH_OFFSET(d_offs); 683 encr_data_len = ENCR_DLEN(d_lens); 684 auth_data_len = AUTH_DLEN(d_lens); 685 if (unlikely(flags & VALID_AAD_BUF)) { 686 /* 687 * We dont support both aad 688 * and auth data separately 689 */ 690 auth_data_len = 0; 691 auth_offset = 0; 692 aad_len = fc_params->aad_buf.size; 693 aad_buf = &fc_params->aad_buf; 694 } 695 cpt_ctx = fc_params->ctx_buf.vaddr; 696 cipher_type = cpt_ctx->enc_cipher; 697 hash_type = cpt_ctx->hash_type; 698 mac_len = cpt_ctx->mac_len; 699 700 /* 701 * Save initial space that followed app data for completion code & 702 * alternate completion code to fall in same cache line as app data 703 */ 704 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 705 m_dma += COMPLETION_CODE_SIZE; 706 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 707 (uint8_t *)m_vaddr; 708 709 c_vaddr = (uint8_t *)m_vaddr + size; 710 c_dma = m_dma + size; 711 size += sizeof(cpt_res_s_t); 712 713 m_vaddr = (uint8_t *)m_vaddr + size; 714 m_dma += size; 715 716 /* start cpt request info struct at 8 byte boundary */ 717 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 718 (uint8_t *)m_vaddr; 719 720 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 721 722 size += sizeof(struct cpt_request_info); 723 m_vaddr = (uint8_t *)m_vaddr + size; 724 m_dma += size; 725 726 if (unlikely(!(flags & VALID_IV_BUF))) { 727 iv_len = 0; 728 iv_offset = ENCR_IV_OFFSET(d_offs); 729 } 730 731 if (unlikely(flags & VALID_AAD_BUF)) { 732 /* 733 * When AAD is given, data above encr_offset is pass through 734 * Since AAD is given as separate pointer and not as offset, 735 * this is a special case as we need to fragment input data 736 * into passthrough + encr_data and then insert AAD in between. 737 */ 738 if (hash_type != GMAC_TYPE) { 739 passthrough_len = encr_offset; 740 auth_offset = passthrough_len + iv_len; 741 encr_offset = passthrough_len + aad_len + iv_len; 742 auth_data_len = aad_len + encr_data_len; 743 } else { 744 passthrough_len = 16 + aad_len; 745 auth_offset = passthrough_len + iv_len; 746 auth_data_len = aad_len; 747 } 748 } else { 749 encr_offset += iv_len; 750 auth_offset += iv_len; 751 } 752 753 /* Encryption */ 754 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 755 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT; 756 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc << 757 CPT_HMAC_FIRST_BIT_POS); 758 759 if (hash_type == GMAC_TYPE) { 760 encr_offset = 0; 761 encr_data_len = 0; 762 } 763 764 auth_dlen = auth_offset + auth_data_len; 765 enc_dlen = encr_data_len + encr_offset; 766 if (unlikely(encr_data_len & 0xf)) { 767 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 768 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 769 encr_offset; 770 else if (likely((cipher_type == AES_CBC) || 771 (cipher_type == AES_ECB))) 772 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 773 encr_offset; 774 } 775 776 if (unlikely(auth_dlen > enc_dlen)) { 777 inputlen = auth_dlen; 778 outputlen = auth_dlen + mac_len; 779 } else { 780 inputlen = enc_dlen; 781 outputlen = enc_dlen + mac_len; 782 } 783 784 if (cpt_ctx->auth_enc != 0) 785 outputlen = enc_dlen; 786 787 /* GP op header */ 788 vq_cmd_w0.s.param1 = encr_data_len; 789 vq_cmd_w0.s.param2 = auth_data_len; 790 /* 791 * In 83XX since we have a limitation of 792 * IV & Offset control word not part of instruction 793 * and need to be part of Data Buffer, we check if 794 * head room is there and then only do the Direct mode processing 795 */ 796 if (likely((flags & SINGLE_BUF_INPLACE) && 797 (flags & SINGLE_BUF_HEADTAILROOM))) { 798 void *dm_vaddr = fc_params->bufs[0].vaddr; 799 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 800 /* 801 * This flag indicates that there is 24 bytes head room and 802 * 8 bytes tail room available, so that we get to do 803 * DIRECT MODE with limitation 804 */ 805 806 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 807 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 808 809 /* DPTR */ 810 req->ist.ei1 = offset_dma; 811 /* RPTR should just exclude offset control word */ 812 req->ist.ei2 = dm_dma_addr - iv_len; 813 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 814 + outputlen - iv_len); 815 816 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 817 818 if (likely(iv_len)) { 819 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 820 + OFF_CTRL_LEN); 821 uint64_t *src = fc_params->iv_buf; 822 dest[0] = src[0]; 823 dest[1] = src[1]; 824 } 825 826 *(uint64_t *)offset_vaddr = 827 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 828 ((uint64_t)iv_offset << 8) | 829 ((uint64_t)auth_offset)); 830 831 } else { 832 uint32_t i, g_size_bytes, s_size_bytes; 833 uint64_t dptr_dma, rptr_dma; 834 sg_comp_t *gather_comp; 835 sg_comp_t *scatter_comp; 836 uint8_t *in_buffer; 837 838 /* This falls under strict SG mode */ 839 offset_vaddr = m_vaddr; 840 offset_dma = m_dma; 841 size = OFF_CTRL_LEN + iv_len; 842 843 m_vaddr = (uint8_t *)m_vaddr + size; 844 m_dma += size; 845 846 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 847 848 if (likely(iv_len)) { 849 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 850 + OFF_CTRL_LEN); 851 uint64_t *src = fc_params->iv_buf; 852 dest[0] = src[0]; 853 dest[1] = src[1]; 854 } 855 856 *(uint64_t *)offset_vaddr = 857 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 858 ((uint64_t)iv_offset << 8) | 859 ((uint64_t)auth_offset)); 860 861 /* DPTR has SG list */ 862 in_buffer = m_vaddr; 863 dptr_dma = m_dma; 864 865 ((uint16_t *)in_buffer)[0] = 0; 866 ((uint16_t *)in_buffer)[1] = 0; 867 868 /* TODO Add error check if space will be sufficient */ 869 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 870 871 /* 872 * Input Gather List 873 */ 874 875 i = 0; 876 877 /* Offset control word that includes iv */ 878 i = fill_sg_comp(gather_comp, i, offset_dma, 879 OFF_CTRL_LEN + iv_len); 880 881 /* Add input data */ 882 size = inputlen - iv_len; 883 if (likely(size)) { 884 uint32_t aad_offset = aad_len ? passthrough_len : 0; 885 886 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 887 i = fill_sg_comp_from_buf_min(gather_comp, i, 888 fc_params->bufs, 889 &size); 890 } else { 891 i = fill_sg_comp_from_iov(gather_comp, i, 892 fc_params->src_iov, 893 0, &size, 894 aad_buf, aad_offset); 895 } 896 897 if (unlikely(size)) { 898 CPT_LOG_DP_ERR("Insufficient buffer space," 899 " size %d needed", size); 900 return; 901 } 902 } 903 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 904 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 905 906 /* 907 * Output Scatter list 908 */ 909 i = 0; 910 scatter_comp = 911 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 912 913 /* Add IV */ 914 if (likely(iv_len)) { 915 i = fill_sg_comp(scatter_comp, i, 916 offset_dma + OFF_CTRL_LEN, 917 iv_len); 918 } 919 920 /* output data or output data + digest*/ 921 if (unlikely(flags & VALID_MAC_BUF)) { 922 size = outputlen - iv_len - mac_len; 923 if (size) { 924 uint32_t aad_offset = 925 aad_len ? passthrough_len : 0; 926 927 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 928 i = fill_sg_comp_from_buf_min( 929 scatter_comp, 930 i, 931 fc_params->bufs, 932 &size); 933 } else { 934 i = fill_sg_comp_from_iov(scatter_comp, 935 i, 936 fc_params->dst_iov, 937 0, 938 &size, 939 aad_buf, 940 aad_offset); 941 } 942 if (unlikely(size)) { 943 CPT_LOG_DP_ERR("Insufficient buffer" 944 " space, size %d needed", 945 size); 946 return; 947 } 948 } 949 /* mac_data */ 950 if (mac_len) { 951 i = fill_sg_comp_from_buf(scatter_comp, i, 952 &fc_params->mac_buf); 953 } 954 } else { 955 /* Output including mac */ 956 size = outputlen - iv_len; 957 if (likely(size)) { 958 uint32_t aad_offset = 959 aad_len ? passthrough_len : 0; 960 961 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 962 i = fill_sg_comp_from_buf_min( 963 scatter_comp, 964 i, 965 fc_params->bufs, 966 &size); 967 } else { 968 i = fill_sg_comp_from_iov(scatter_comp, 969 i, 970 fc_params->dst_iov, 971 0, 972 &size, 973 aad_buf, 974 aad_offset); 975 } 976 if (unlikely(size)) { 977 CPT_LOG_DP_ERR("Insufficient buffer" 978 " space, size %d needed", 979 size); 980 return; 981 } 982 } 983 } 984 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 985 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 986 987 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 988 989 /* This is DPTR len incase of SG mode */ 990 vq_cmd_w0.s.dlen = size; 991 992 m_vaddr = (uint8_t *)m_vaddr + size; 993 m_dma += size; 994 995 /* cpt alternate completion address saved earlier */ 996 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 997 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 998 rptr_dma = c_dma - 8; 999 1000 req->ist.ei1 = dptr_dma; 1001 req->ist.ei2 = rptr_dma; 1002 } 1003 1004 if (unlikely((encr_offset >> 16) || 1005 (iv_offset >> 8) || 1006 (auth_offset >> 8))) { 1007 CPT_LOG_DP_ERR("Offset not supported"); 1008 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1009 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1010 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1011 return; 1012 } 1013 1014 /* 16 byte aligned cpt res address */ 1015 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1016 *req->completion_addr = COMPLETION_CODE_INIT; 1017 req->comp_baddr = c_dma; 1018 1019 /* Fill microcode part of instruction */ 1020 req->ist.ei0 = vq_cmd_w0.u64; 1021 1022 req->op = op; 1023 1024 *prep_req = req; 1025 return; 1026 } 1027 1028 static __rte_always_inline void 1029 cpt_dec_hmac_prep(uint32_t flags, 1030 uint64_t d_offs, 1031 uint64_t d_lens, 1032 fc_params_t *fc_params, 1033 void *op, 1034 void **prep_req) 1035 { 1036 uint32_t iv_offset = 0, size; 1037 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1038 struct cpt_ctx *cpt_ctx; 1039 int32_t hash_type, mac_len; 1040 uint8_t iv_len = 16; 1041 struct cpt_request_info *req; 1042 buf_ptr_t *meta_p, *aad_buf = NULL; 1043 uint32_t encr_offset, auth_offset; 1044 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1045 uint32_t passthrough_len = 0; 1046 void *m_vaddr, *offset_vaddr; 1047 uint64_t m_dma, offset_dma; 1048 vq_cmd_word0_t vq_cmd_w0; 1049 void *c_vaddr; 1050 uint64_t c_dma; 1051 1052 meta_p = &fc_params->meta_buf; 1053 m_vaddr = meta_p->vaddr; 1054 m_dma = meta_p->dma_addr; 1055 1056 encr_offset = ENCR_OFFSET(d_offs); 1057 auth_offset = AUTH_OFFSET(d_offs); 1058 encr_data_len = ENCR_DLEN(d_lens); 1059 auth_data_len = AUTH_DLEN(d_lens); 1060 1061 if (unlikely(flags & VALID_AAD_BUF)) { 1062 /* 1063 * We dont support both aad 1064 * and auth data separately 1065 */ 1066 auth_data_len = 0; 1067 auth_offset = 0; 1068 aad_len = fc_params->aad_buf.size; 1069 aad_buf = &fc_params->aad_buf; 1070 } 1071 1072 cpt_ctx = fc_params->ctx_buf.vaddr; 1073 hash_type = cpt_ctx->hash_type; 1074 mac_len = cpt_ctx->mac_len; 1075 1076 if (unlikely(!(flags & VALID_IV_BUF))) { 1077 iv_len = 0; 1078 iv_offset = ENCR_IV_OFFSET(d_offs); 1079 } 1080 1081 if (unlikely(flags & VALID_AAD_BUF)) { 1082 /* 1083 * When AAD is given, data above encr_offset is pass through 1084 * Since AAD is given as separate pointer and not as offset, 1085 * this is a special case as we need to fragment input data 1086 * into passthrough + encr_data and then insert AAD in between. 1087 */ 1088 if (hash_type != GMAC_TYPE) { 1089 passthrough_len = encr_offset; 1090 auth_offset = passthrough_len + iv_len; 1091 encr_offset = passthrough_len + aad_len + iv_len; 1092 auth_data_len = aad_len + encr_data_len; 1093 } else { 1094 passthrough_len = 16 + aad_len; 1095 auth_offset = passthrough_len + iv_len; 1096 auth_data_len = aad_len; 1097 } 1098 } else { 1099 encr_offset += iv_len; 1100 auth_offset += iv_len; 1101 } 1102 1103 /* 1104 * Save initial space that followed app data for completion code & 1105 * alternate completion code to fall in same cache line as app data 1106 */ 1107 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1108 m_dma += COMPLETION_CODE_SIZE; 1109 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1110 (uint8_t *)m_vaddr; 1111 c_vaddr = (uint8_t *)m_vaddr + size; 1112 c_dma = m_dma + size; 1113 size += sizeof(cpt_res_s_t); 1114 1115 m_vaddr = (uint8_t *)m_vaddr + size; 1116 m_dma += size; 1117 1118 /* start cpt request info structure at 8 byte alignment */ 1119 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1120 (uint8_t *)m_vaddr; 1121 1122 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1123 1124 size += sizeof(struct cpt_request_info); 1125 m_vaddr = (uint8_t *)m_vaddr + size; 1126 m_dma += size; 1127 1128 /* Decryption */ 1129 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 1130 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT; 1131 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth << 1132 CPT_HMAC_FIRST_BIT_POS); 1133 1134 if (hash_type == GMAC_TYPE) { 1135 encr_offset = 0; 1136 encr_data_len = 0; 1137 } 1138 1139 enc_dlen = encr_offset + encr_data_len; 1140 auth_dlen = auth_offset + auth_data_len; 1141 1142 if (auth_dlen > enc_dlen) { 1143 inputlen = auth_dlen + mac_len; 1144 outputlen = auth_dlen; 1145 } else { 1146 inputlen = enc_dlen + mac_len; 1147 outputlen = enc_dlen; 1148 } 1149 1150 if (cpt_ctx->dec_auth != 0) 1151 outputlen = inputlen = enc_dlen; 1152 1153 vq_cmd_w0.s.param1 = encr_data_len; 1154 vq_cmd_w0.s.param2 = auth_data_len; 1155 1156 /* 1157 * In 83XX since we have a limitation of 1158 * IV & Offset control word not part of instruction 1159 * and need to be part of Data Buffer, we check if 1160 * head room is there and then only do the Direct mode processing 1161 */ 1162 if (likely((flags & SINGLE_BUF_INPLACE) && 1163 (flags & SINGLE_BUF_HEADTAILROOM))) { 1164 void *dm_vaddr = fc_params->bufs[0].vaddr; 1165 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1166 /* 1167 * This flag indicates that there is 24 bytes head room and 1168 * 8 bytes tail room available, so that we get to do 1169 * DIRECT MODE with limitation 1170 */ 1171 1172 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1173 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1174 req->ist.ei1 = offset_dma; 1175 1176 /* RPTR should just exclude offset control word */ 1177 req->ist.ei2 = dm_dma_addr - iv_len; 1178 1179 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1180 outputlen - iv_len); 1181 /* since this is decryption, 1182 * don't touch the content of 1183 * alternate ccode space as it contains 1184 * hmac. 1185 */ 1186 1187 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1188 1189 if (likely(iv_len)) { 1190 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1191 OFF_CTRL_LEN); 1192 uint64_t *src = fc_params->iv_buf; 1193 dest[0] = src[0]; 1194 dest[1] = src[1]; 1195 } 1196 1197 if (unlikely((encr_offset >> 16) || 1198 (iv_offset >> 8) || 1199 (auth_offset >> 8))) { 1200 CPT_LOG_DP_ERR("Offset not supported"); 1201 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1202 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1203 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1204 return; 1205 } 1206 1207 *(uint64_t *)offset_vaddr = 1208 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1209 ((uint64_t)iv_offset << 8) | 1210 ((uint64_t)auth_offset)); 1211 1212 } else { 1213 uint64_t dptr_dma, rptr_dma; 1214 uint32_t g_size_bytes, s_size_bytes; 1215 sg_comp_t *gather_comp; 1216 sg_comp_t *scatter_comp; 1217 uint8_t *in_buffer; 1218 uint8_t i = 0; 1219 1220 /* This falls under strict SG mode */ 1221 offset_vaddr = m_vaddr; 1222 offset_dma = m_dma; 1223 size = OFF_CTRL_LEN + iv_len; 1224 1225 m_vaddr = (uint8_t *)m_vaddr + size; 1226 m_dma += size; 1227 1228 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1229 1230 if (likely(iv_len)) { 1231 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1232 OFF_CTRL_LEN); 1233 uint64_t *src = fc_params->iv_buf; 1234 dest[0] = src[0]; 1235 dest[1] = src[1]; 1236 } 1237 1238 if (unlikely((encr_offset >> 16) || 1239 (iv_offset >> 8) || 1240 (auth_offset >> 8))) { 1241 CPT_LOG_DP_ERR("Offset not supported"); 1242 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1243 CPT_LOG_DP_ERR("iv_offset : %d", iv_offset); 1244 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1245 return; 1246 } 1247 1248 *(uint64_t *)offset_vaddr = 1249 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1250 ((uint64_t)iv_offset << 8) | 1251 ((uint64_t)auth_offset)); 1252 1253 /* DPTR has SG list */ 1254 in_buffer = m_vaddr; 1255 dptr_dma = m_dma; 1256 1257 ((uint16_t *)in_buffer)[0] = 0; 1258 ((uint16_t *)in_buffer)[1] = 0; 1259 1260 /* TODO Add error check if space will be sufficient */ 1261 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1262 1263 /* 1264 * Input Gather List 1265 */ 1266 i = 0; 1267 1268 /* Offset control word that includes iv */ 1269 i = fill_sg_comp(gather_comp, i, offset_dma, 1270 OFF_CTRL_LEN + iv_len); 1271 1272 /* Add input data */ 1273 if (flags & VALID_MAC_BUF) { 1274 size = inputlen - iv_len - mac_len; 1275 if (size) { 1276 /* input data only */ 1277 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1278 i = fill_sg_comp_from_buf_min( 1279 gather_comp, i, 1280 fc_params->bufs, 1281 &size); 1282 } else { 1283 uint32_t aad_offset = aad_len ? 1284 passthrough_len : 0; 1285 1286 i = fill_sg_comp_from_iov(gather_comp, 1287 i, 1288 fc_params->src_iov, 1289 0, &size, 1290 aad_buf, 1291 aad_offset); 1292 } 1293 if (unlikely(size)) { 1294 CPT_LOG_DP_ERR("Insufficient buffer" 1295 " space, size %d needed", 1296 size); 1297 return; 1298 } 1299 } 1300 1301 /* mac data */ 1302 if (mac_len) { 1303 i = fill_sg_comp_from_buf(gather_comp, i, 1304 &fc_params->mac_buf); 1305 } 1306 } else { 1307 /* input data + mac */ 1308 size = inputlen - iv_len; 1309 if (size) { 1310 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1311 i = fill_sg_comp_from_buf_min( 1312 gather_comp, i, 1313 fc_params->bufs, 1314 &size); 1315 } else { 1316 uint32_t aad_offset = aad_len ? 1317 passthrough_len : 0; 1318 1319 if (unlikely(!fc_params->src_iov)) { 1320 CPT_LOG_DP_ERR("Bad input args"); 1321 return; 1322 } 1323 1324 i = fill_sg_comp_from_iov( 1325 gather_comp, i, 1326 fc_params->src_iov, 1327 0, &size, 1328 aad_buf, 1329 aad_offset); 1330 } 1331 1332 if (unlikely(size)) { 1333 CPT_LOG_DP_ERR("Insufficient buffer" 1334 " space, size %d needed", 1335 size); 1336 return; 1337 } 1338 } 1339 } 1340 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1341 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1342 1343 /* 1344 * Output Scatter List 1345 */ 1346 1347 i = 0; 1348 scatter_comp = 1349 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1350 1351 /* Add iv */ 1352 if (iv_len) { 1353 i = fill_sg_comp(scatter_comp, i, 1354 offset_dma + OFF_CTRL_LEN, 1355 iv_len); 1356 } 1357 1358 /* Add output data */ 1359 size = outputlen - iv_len; 1360 if (size) { 1361 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1362 /* handle single buffer here */ 1363 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1364 fc_params->bufs, 1365 &size); 1366 } else { 1367 uint32_t aad_offset = aad_len ? 1368 passthrough_len : 0; 1369 1370 if (unlikely(!fc_params->dst_iov)) { 1371 CPT_LOG_DP_ERR("Bad input args"); 1372 return; 1373 } 1374 1375 i = fill_sg_comp_from_iov(scatter_comp, i, 1376 fc_params->dst_iov, 0, 1377 &size, aad_buf, 1378 aad_offset); 1379 } 1380 1381 if (unlikely(size)) { 1382 CPT_LOG_DP_ERR("Insufficient buffer space," 1383 " size %d needed", size); 1384 return; 1385 } 1386 } 1387 1388 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1389 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1390 1391 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1392 1393 /* This is DPTR len incase of SG mode */ 1394 vq_cmd_w0.s.dlen = size; 1395 1396 m_vaddr = (uint8_t *)m_vaddr + size; 1397 m_dma += size; 1398 1399 /* cpt alternate completion address saved earlier */ 1400 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1401 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1402 rptr_dma = c_dma - 8; 1403 size += COMPLETION_CODE_SIZE; 1404 1405 req->ist.ei1 = dptr_dma; 1406 req->ist.ei2 = rptr_dma; 1407 } 1408 1409 /* 16 byte aligned cpt res address */ 1410 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1411 *req->completion_addr = COMPLETION_CODE_INIT; 1412 req->comp_baddr = c_dma; 1413 1414 /* Fill microcode part of instruction */ 1415 req->ist.ei0 = vq_cmd_w0.u64; 1416 1417 req->op = op; 1418 1419 *prep_req = req; 1420 return; 1421 } 1422 1423 static __rte_always_inline void 1424 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1425 uint64_t d_offs, 1426 uint64_t d_lens, 1427 fc_params_t *params, 1428 void *op, 1429 void **prep_req) 1430 { 1431 uint32_t size; 1432 int32_t inputlen, outputlen; 1433 struct cpt_ctx *cpt_ctx; 1434 uint32_t mac_len = 0; 1435 uint8_t snow3g, j; 1436 struct cpt_request_info *req; 1437 buf_ptr_t *buf_p; 1438 uint32_t encr_offset = 0, auth_offset = 0; 1439 uint32_t encr_data_len = 0, auth_data_len = 0; 1440 int flags, iv_len = 16; 1441 void *m_vaddr, *c_vaddr; 1442 uint64_t m_dma, c_dma, offset_ctrl; 1443 uint64_t *offset_vaddr, offset_dma; 1444 uint32_t *iv_s, iv[4]; 1445 vq_cmd_word0_t vq_cmd_w0; 1446 1447 buf_p = ¶ms->meta_buf; 1448 m_vaddr = buf_p->vaddr; 1449 m_dma = buf_p->dma_addr; 1450 1451 cpt_ctx = params->ctx_buf.vaddr; 1452 flags = cpt_ctx->zsk_flags; 1453 mac_len = cpt_ctx->mac_len; 1454 snow3g = cpt_ctx->snow3g; 1455 1456 /* 1457 * Save initial space that followed app data for completion code & 1458 * alternate completion code to fall in same cache line as app data 1459 */ 1460 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1461 m_dma += COMPLETION_CODE_SIZE; 1462 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1463 (uint8_t *)m_vaddr; 1464 1465 c_vaddr = (uint8_t *)m_vaddr + size; 1466 c_dma = m_dma + size; 1467 size += sizeof(cpt_res_s_t); 1468 1469 m_vaddr = (uint8_t *)m_vaddr + size; 1470 m_dma += size; 1471 1472 /* Reserve memory for cpt request info */ 1473 req = m_vaddr; 1474 1475 size = sizeof(struct cpt_request_info); 1476 m_vaddr = (uint8_t *)m_vaddr + size; 1477 m_dma += size; 1478 1479 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1480 1481 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1482 1483 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1484 (0 << 3) | (flags & 0x7)); 1485 1486 if (flags == 0x1) { 1487 /* 1488 * Microcode expects offsets in bytes 1489 * TODO: Rounding off 1490 */ 1491 auth_data_len = AUTH_DLEN(d_lens); 1492 1493 /* EIA3 or UIA2 */ 1494 auth_offset = AUTH_OFFSET(d_offs); 1495 auth_offset = auth_offset / 8; 1496 1497 /* consider iv len */ 1498 auth_offset += iv_len; 1499 1500 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1501 outputlen = mac_len; 1502 1503 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1504 1505 } else { 1506 /* EEA3 or UEA2 */ 1507 /* 1508 * Microcode expects offsets in bytes 1509 * TODO: Rounding off 1510 */ 1511 encr_data_len = ENCR_DLEN(d_lens); 1512 1513 encr_offset = ENCR_OFFSET(d_offs); 1514 encr_offset = encr_offset / 8; 1515 /* consider iv len */ 1516 encr_offset += iv_len; 1517 1518 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1519 outputlen = inputlen; 1520 1521 /* iv offset is 0 */ 1522 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1523 } 1524 1525 if (unlikely((encr_offset >> 16) || 1526 (auth_offset >> 8))) { 1527 CPT_LOG_DP_ERR("Offset not supported"); 1528 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1529 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 1530 return; 1531 } 1532 1533 /* IV */ 1534 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1535 params->iv_buf; 1536 1537 if (snow3g) { 1538 /* 1539 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1540 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1541 */ 1542 1543 for (j = 0; j < 4; j++) 1544 iv[j] = iv_s[3 - j]; 1545 } else { 1546 /* ZUC doesn't need a swap */ 1547 for (j = 0; j < 4; j++) 1548 iv[j] = iv_s[j]; 1549 } 1550 1551 /* 1552 * GP op header, lengths are expected in bits. 1553 */ 1554 vq_cmd_w0.s.param1 = encr_data_len; 1555 vq_cmd_w0.s.param2 = auth_data_len; 1556 1557 /* 1558 * In 83XX since we have a limitation of 1559 * IV & Offset control word not part of instruction 1560 * and need to be part of Data Buffer, we check if 1561 * head room is there and then only do the Direct mode processing 1562 */ 1563 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1564 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1565 void *dm_vaddr = params->bufs[0].vaddr; 1566 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1567 /* 1568 * This flag indicates that there is 24 bytes head room and 1569 * 8 bytes tail room available, so that we get to do 1570 * DIRECT MODE with limitation 1571 */ 1572 1573 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1574 OFF_CTRL_LEN - iv_len); 1575 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1576 1577 /* DPTR */ 1578 req->ist.ei1 = offset_dma; 1579 /* RPTR should just exclude offset control word */ 1580 req->ist.ei2 = dm_dma_addr - iv_len; 1581 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1582 + outputlen - iv_len); 1583 1584 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1585 1586 if (likely(iv_len)) { 1587 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1588 + OFF_CTRL_LEN); 1589 memcpy(iv_d, iv, 16); 1590 } 1591 1592 *offset_vaddr = offset_ctrl; 1593 } else { 1594 uint32_t i, g_size_bytes, s_size_bytes; 1595 uint64_t dptr_dma, rptr_dma; 1596 sg_comp_t *gather_comp; 1597 sg_comp_t *scatter_comp; 1598 uint8_t *in_buffer; 1599 uint32_t *iv_d; 1600 1601 /* save space for iv */ 1602 offset_vaddr = m_vaddr; 1603 offset_dma = m_dma; 1604 1605 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1606 m_dma += OFF_CTRL_LEN + iv_len; 1607 1608 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1609 1610 /* DPTR has SG list */ 1611 in_buffer = m_vaddr; 1612 dptr_dma = m_dma; 1613 1614 ((uint16_t *)in_buffer)[0] = 0; 1615 ((uint16_t *)in_buffer)[1] = 0; 1616 1617 /* TODO Add error check if space will be sufficient */ 1618 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1619 1620 /* 1621 * Input Gather List 1622 */ 1623 i = 0; 1624 1625 /* Offset control word followed by iv */ 1626 1627 i = fill_sg_comp(gather_comp, i, offset_dma, 1628 OFF_CTRL_LEN + iv_len); 1629 1630 /* iv offset is 0 */ 1631 *offset_vaddr = offset_ctrl; 1632 1633 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1634 memcpy(iv_d, iv, 16); 1635 1636 /* input data */ 1637 size = inputlen - iv_len; 1638 if (size) { 1639 i = fill_sg_comp_from_iov(gather_comp, i, 1640 params->src_iov, 1641 0, &size, NULL, 0); 1642 if (unlikely(size)) { 1643 CPT_LOG_DP_ERR("Insufficient buffer space," 1644 " size %d needed", size); 1645 return; 1646 } 1647 } 1648 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1649 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1650 1651 /* 1652 * Output Scatter List 1653 */ 1654 1655 i = 0; 1656 scatter_comp = 1657 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1658 1659 if (flags == 0x1) { 1660 /* IV in SLIST only for EEA3 & UEA2 */ 1661 iv_len = 0; 1662 } 1663 1664 if (iv_len) { 1665 i = fill_sg_comp(scatter_comp, i, 1666 offset_dma + OFF_CTRL_LEN, iv_len); 1667 } 1668 1669 /* Add output data */ 1670 if (req_flags & VALID_MAC_BUF) { 1671 size = outputlen - iv_len - mac_len; 1672 if (size) { 1673 i = fill_sg_comp_from_iov(scatter_comp, i, 1674 params->dst_iov, 0, 1675 &size, NULL, 0); 1676 1677 if (unlikely(size)) { 1678 CPT_LOG_DP_ERR("Insufficient buffer space," 1679 " size %d needed", size); 1680 return; 1681 } 1682 } 1683 1684 /* mac data */ 1685 if (mac_len) { 1686 i = fill_sg_comp_from_buf(scatter_comp, i, 1687 ¶ms->mac_buf); 1688 } 1689 } else { 1690 /* Output including mac */ 1691 size = outputlen - iv_len; 1692 if (size) { 1693 i = fill_sg_comp_from_iov(scatter_comp, i, 1694 params->dst_iov, 0, 1695 &size, NULL, 0); 1696 1697 if (unlikely(size)) { 1698 CPT_LOG_DP_ERR("Insufficient buffer space," 1699 " size %d needed", size); 1700 return; 1701 } 1702 } 1703 } 1704 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1705 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1706 1707 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1708 1709 /* This is DPTR len incase of SG mode */ 1710 vq_cmd_w0.s.dlen = size; 1711 1712 m_vaddr = (uint8_t *)m_vaddr + size; 1713 m_dma += size; 1714 1715 /* cpt alternate completion address saved earlier */ 1716 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1717 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1718 rptr_dma = c_dma - 8; 1719 1720 req->ist.ei1 = dptr_dma; 1721 req->ist.ei2 = rptr_dma; 1722 } 1723 1724 /* 16 byte aligned cpt res address */ 1725 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1726 *req->completion_addr = COMPLETION_CODE_INIT; 1727 req->comp_baddr = c_dma; 1728 1729 /* Fill microcode part of instruction */ 1730 req->ist.ei0 = vq_cmd_w0.u64; 1731 1732 req->op = op; 1733 1734 *prep_req = req; 1735 return; 1736 } 1737 1738 static __rte_always_inline void 1739 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1740 uint64_t d_offs, 1741 uint64_t d_lens, 1742 fc_params_t *params, 1743 void *op, 1744 void **prep_req) 1745 { 1746 uint32_t size; 1747 int32_t inputlen = 0, outputlen; 1748 struct cpt_ctx *cpt_ctx; 1749 uint8_t snow3g, iv_len = 16; 1750 struct cpt_request_info *req; 1751 buf_ptr_t *buf_p; 1752 uint32_t encr_offset; 1753 uint32_t encr_data_len; 1754 int flags; 1755 void *m_vaddr, *c_vaddr; 1756 uint64_t m_dma, c_dma; 1757 uint64_t *offset_vaddr, offset_dma; 1758 uint32_t *iv_s, iv[4], j; 1759 vq_cmd_word0_t vq_cmd_w0; 1760 1761 buf_p = ¶ms->meta_buf; 1762 m_vaddr = buf_p->vaddr; 1763 m_dma = buf_p->dma_addr; 1764 1765 /* 1766 * Microcode expects offsets in bytes 1767 * TODO: Rounding off 1768 */ 1769 encr_offset = ENCR_OFFSET(d_offs) / 8; 1770 encr_data_len = ENCR_DLEN(d_lens); 1771 1772 cpt_ctx = params->ctx_buf.vaddr; 1773 flags = cpt_ctx->zsk_flags; 1774 snow3g = cpt_ctx->snow3g; 1775 /* 1776 * Save initial space that followed app data for completion code & 1777 * alternate completion code to fall in same cache line as app data 1778 */ 1779 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1780 m_dma += COMPLETION_CODE_SIZE; 1781 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1782 (uint8_t *)m_vaddr; 1783 1784 c_vaddr = (uint8_t *)m_vaddr + size; 1785 c_dma = m_dma + size; 1786 size += sizeof(cpt_res_s_t); 1787 1788 m_vaddr = (uint8_t *)m_vaddr + size; 1789 m_dma += size; 1790 1791 /* Reserve memory for cpt request info */ 1792 req = m_vaddr; 1793 1794 size = sizeof(struct cpt_request_info); 1795 m_vaddr = (uint8_t *)m_vaddr + size; 1796 m_dma += size; 1797 1798 vq_cmd_w0.u64 = 0; 1799 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1800 1801 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1802 1803 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1804 (0 << 3) | (flags & 0x7)); 1805 1806 /* consider iv len */ 1807 encr_offset += iv_len; 1808 1809 inputlen = encr_offset + 1810 (RTE_ALIGN(encr_data_len, 8) / 8); 1811 outputlen = inputlen; 1812 1813 /* IV */ 1814 iv_s = params->iv_buf; 1815 if (snow3g) { 1816 /* 1817 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1818 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1819 */ 1820 1821 for (j = 0; j < 4; j++) 1822 iv[j] = iv_s[3 - j]; 1823 } else { 1824 /* ZUC doesn't need a swap */ 1825 for (j = 0; j < 4; j++) 1826 iv[j] = iv_s[j]; 1827 } 1828 1829 /* 1830 * GP op header, lengths are expected in bits. 1831 */ 1832 vq_cmd_w0.s.param1 = encr_data_len; 1833 1834 /* 1835 * In 83XX since we have a limitation of 1836 * IV & Offset control word not part of instruction 1837 * and need to be part of Data Buffer, we check if 1838 * head room is there and then only do the Direct mode processing 1839 */ 1840 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1841 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1842 void *dm_vaddr = params->bufs[0].vaddr; 1843 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1844 /* 1845 * This flag indicates that there is 24 bytes head room and 1846 * 8 bytes tail room available, so that we get to do 1847 * DIRECT MODE with limitation 1848 */ 1849 1850 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1851 OFF_CTRL_LEN - iv_len); 1852 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1853 1854 /* DPTR */ 1855 req->ist.ei1 = offset_dma; 1856 /* RPTR should just exclude offset control word */ 1857 req->ist.ei2 = dm_dma_addr - iv_len; 1858 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1859 + outputlen - iv_len); 1860 1861 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1862 1863 if (likely(iv_len)) { 1864 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1865 + OFF_CTRL_LEN); 1866 memcpy(iv_d, iv, 16); 1867 } 1868 1869 /* iv offset is 0 */ 1870 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1871 } else { 1872 uint32_t i, g_size_bytes, s_size_bytes; 1873 uint64_t dptr_dma, rptr_dma; 1874 sg_comp_t *gather_comp; 1875 sg_comp_t *scatter_comp; 1876 uint8_t *in_buffer; 1877 uint32_t *iv_d; 1878 1879 /* save space for offset and iv... */ 1880 offset_vaddr = m_vaddr; 1881 offset_dma = m_dma; 1882 1883 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1884 m_dma += OFF_CTRL_LEN + iv_len; 1885 1886 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1887 1888 /* DPTR has SG list */ 1889 in_buffer = m_vaddr; 1890 dptr_dma = m_dma; 1891 1892 ((uint16_t *)in_buffer)[0] = 0; 1893 ((uint16_t *)in_buffer)[1] = 0; 1894 1895 /* TODO Add error check if space will be sufficient */ 1896 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1897 1898 /* 1899 * Input Gather List 1900 */ 1901 i = 0; 1902 1903 /* Offset control word */ 1904 1905 /* iv offset is 0 */ 1906 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1907 1908 i = fill_sg_comp(gather_comp, i, offset_dma, 1909 OFF_CTRL_LEN + iv_len); 1910 1911 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1912 memcpy(iv_d, iv, 16); 1913 1914 /* Add input data */ 1915 size = inputlen - iv_len; 1916 if (size) { 1917 i = fill_sg_comp_from_iov(gather_comp, i, 1918 params->src_iov, 1919 0, &size, NULL, 0); 1920 if (unlikely(size)) { 1921 CPT_LOG_DP_ERR("Insufficient buffer space," 1922 " size %d needed", size); 1923 return; 1924 } 1925 } 1926 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1927 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1928 1929 /* 1930 * Output Scatter List 1931 */ 1932 1933 i = 0; 1934 scatter_comp = 1935 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1936 1937 /* IV */ 1938 i = fill_sg_comp(scatter_comp, i, 1939 offset_dma + OFF_CTRL_LEN, 1940 iv_len); 1941 1942 /* Add output data */ 1943 size = outputlen - iv_len; 1944 if (size) { 1945 i = fill_sg_comp_from_iov(scatter_comp, i, 1946 params->dst_iov, 0, 1947 &size, NULL, 0); 1948 1949 if (unlikely(size)) { 1950 CPT_LOG_DP_ERR("Insufficient buffer space," 1951 " size %d needed", size); 1952 return; 1953 } 1954 } 1955 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1956 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1957 1958 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1959 1960 /* This is DPTR len incase of SG mode */ 1961 vq_cmd_w0.s.dlen = size; 1962 1963 m_vaddr = (uint8_t *)m_vaddr + size; 1964 m_dma += size; 1965 1966 /* cpt alternate completion address saved earlier */ 1967 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1968 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1969 rptr_dma = c_dma - 8; 1970 1971 req->ist.ei1 = dptr_dma; 1972 req->ist.ei2 = rptr_dma; 1973 } 1974 1975 if (unlikely((encr_offset >> 16))) { 1976 CPT_LOG_DP_ERR("Offset not supported"); 1977 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 1978 return; 1979 } 1980 1981 /* 16 byte aligned cpt res address */ 1982 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1983 *req->completion_addr = COMPLETION_CODE_INIT; 1984 req->comp_baddr = c_dma; 1985 1986 /* Fill microcode part of instruction */ 1987 req->ist.ei0 = vq_cmd_w0.u64; 1988 1989 req->op = op; 1990 1991 *prep_req = req; 1992 return; 1993 } 1994 1995 static __rte_always_inline void 1996 cpt_kasumi_enc_prep(uint32_t req_flags, 1997 uint64_t d_offs, 1998 uint64_t d_lens, 1999 fc_params_t *params, 2000 void *op, 2001 void **prep_req) 2002 { 2003 uint32_t size; 2004 int32_t inputlen = 0, outputlen = 0; 2005 struct cpt_ctx *cpt_ctx; 2006 uint32_t mac_len = 0; 2007 uint8_t i = 0; 2008 struct cpt_request_info *req; 2009 buf_ptr_t *buf_p; 2010 uint32_t encr_offset, auth_offset; 2011 uint32_t encr_data_len, auth_data_len; 2012 int flags; 2013 uint8_t *iv_s, *iv_d, iv_len = 8; 2014 uint8_t dir = 0; 2015 void *m_vaddr, *c_vaddr; 2016 uint64_t m_dma, c_dma; 2017 uint64_t *offset_vaddr, offset_dma; 2018 vq_cmd_word0_t vq_cmd_w0; 2019 uint8_t *in_buffer; 2020 uint32_t g_size_bytes, s_size_bytes; 2021 uint64_t dptr_dma, rptr_dma; 2022 sg_comp_t *gather_comp; 2023 sg_comp_t *scatter_comp; 2024 2025 buf_p = ¶ms->meta_buf; 2026 m_vaddr = buf_p->vaddr; 2027 m_dma = buf_p->dma_addr; 2028 2029 encr_offset = ENCR_OFFSET(d_offs) / 8; 2030 auth_offset = AUTH_OFFSET(d_offs) / 8; 2031 encr_data_len = ENCR_DLEN(d_lens); 2032 auth_data_len = AUTH_DLEN(d_lens); 2033 2034 cpt_ctx = params->ctx_buf.vaddr; 2035 flags = cpt_ctx->zsk_flags; 2036 mac_len = cpt_ctx->mac_len; 2037 2038 if (flags == 0x0) 2039 iv_s = params->iv_buf; 2040 else 2041 iv_s = params->auth_iv_buf; 2042 2043 dir = iv_s[8] & 0x1; 2044 2045 /* 2046 * Save initial space that followed app data for completion code & 2047 * alternate completion code to fall in same cache line as app data 2048 */ 2049 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2050 m_dma += COMPLETION_CODE_SIZE; 2051 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2052 (uint8_t *)m_vaddr; 2053 2054 c_vaddr = (uint8_t *)m_vaddr + size; 2055 c_dma = m_dma + size; 2056 size += sizeof(cpt_res_s_t); 2057 2058 m_vaddr = (uint8_t *)m_vaddr + size; 2059 m_dma += size; 2060 2061 /* Reserve memory for cpt request info */ 2062 req = m_vaddr; 2063 2064 size = sizeof(struct cpt_request_info); 2065 m_vaddr = (uint8_t *)m_vaddr + size; 2066 m_dma += size; 2067 2068 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2069 2070 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2071 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2072 (dir << 4) | (0 << 3) | (flags & 0x7)); 2073 2074 /* 2075 * GP op header, lengths are expected in bits. 2076 */ 2077 vq_cmd_w0.s.param1 = encr_data_len; 2078 vq_cmd_w0.s.param2 = auth_data_len; 2079 2080 /* consider iv len */ 2081 if (flags == 0x0) { 2082 encr_offset += iv_len; 2083 auth_offset += iv_len; 2084 } 2085 2086 /* save space for offset ctrl and iv */ 2087 offset_vaddr = m_vaddr; 2088 offset_dma = m_dma; 2089 2090 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2091 m_dma += OFF_CTRL_LEN + iv_len; 2092 2093 /* DPTR has SG list */ 2094 in_buffer = m_vaddr; 2095 dptr_dma = m_dma; 2096 2097 ((uint16_t *)in_buffer)[0] = 0; 2098 ((uint16_t *)in_buffer)[1] = 0; 2099 2100 /* TODO Add error check if space will be sufficient */ 2101 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2102 2103 /* 2104 * Input Gather List 2105 */ 2106 i = 0; 2107 2108 /* Offset control word followed by iv */ 2109 2110 if (flags == 0x0) { 2111 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2112 outputlen = inputlen; 2113 /* iv offset is 0 */ 2114 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2115 if (unlikely((encr_offset >> 16))) { 2116 CPT_LOG_DP_ERR("Offset not supported"); 2117 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 2118 return; 2119 } 2120 } else { 2121 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2122 outputlen = mac_len; 2123 /* iv offset is 0 */ 2124 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2125 if (unlikely((auth_offset >> 8))) { 2126 CPT_LOG_DP_ERR("Offset not supported"); 2127 CPT_LOG_DP_ERR("auth_offset: %d", auth_offset); 2128 return; 2129 } 2130 } 2131 2132 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2133 2134 /* IV */ 2135 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2136 memcpy(iv_d, iv_s, iv_len); 2137 2138 /* input data */ 2139 size = inputlen - iv_len; 2140 if (size) { 2141 i = fill_sg_comp_from_iov(gather_comp, i, 2142 params->src_iov, 0, 2143 &size, NULL, 0); 2144 2145 if (unlikely(size)) { 2146 CPT_LOG_DP_ERR("Insufficient buffer space," 2147 " size %d needed", size); 2148 return; 2149 } 2150 } 2151 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2152 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2153 2154 /* 2155 * Output Scatter List 2156 */ 2157 2158 i = 0; 2159 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2160 2161 if (flags == 0x1) { 2162 /* IV in SLIST only for F8 */ 2163 iv_len = 0; 2164 } 2165 2166 /* IV */ 2167 if (iv_len) { 2168 i = fill_sg_comp(scatter_comp, i, 2169 offset_dma + OFF_CTRL_LEN, 2170 iv_len); 2171 } 2172 2173 /* Add output data */ 2174 if (req_flags & VALID_MAC_BUF) { 2175 size = outputlen - iv_len - mac_len; 2176 if (size) { 2177 i = fill_sg_comp_from_iov(scatter_comp, i, 2178 params->dst_iov, 0, 2179 &size, NULL, 0); 2180 2181 if (unlikely(size)) { 2182 CPT_LOG_DP_ERR("Insufficient buffer space," 2183 " size %d needed", size); 2184 return; 2185 } 2186 } 2187 2188 /* mac data */ 2189 if (mac_len) { 2190 i = fill_sg_comp_from_buf(scatter_comp, i, 2191 ¶ms->mac_buf); 2192 } 2193 } else { 2194 /* Output including mac */ 2195 size = outputlen - iv_len; 2196 if (size) { 2197 i = fill_sg_comp_from_iov(scatter_comp, i, 2198 params->dst_iov, 0, 2199 &size, NULL, 0); 2200 2201 if (unlikely(size)) { 2202 CPT_LOG_DP_ERR("Insufficient buffer space," 2203 " size %d needed", size); 2204 return; 2205 } 2206 } 2207 } 2208 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2209 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2210 2211 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2212 2213 /* This is DPTR len incase of SG mode */ 2214 vq_cmd_w0.s.dlen = size; 2215 2216 m_vaddr = (uint8_t *)m_vaddr + size; 2217 m_dma += size; 2218 2219 /* cpt alternate completion address saved earlier */ 2220 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2221 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2222 rptr_dma = c_dma - 8; 2223 2224 req->ist.ei1 = dptr_dma; 2225 req->ist.ei2 = rptr_dma; 2226 2227 /* 16 byte aligned cpt res address */ 2228 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2229 *req->completion_addr = COMPLETION_CODE_INIT; 2230 req->comp_baddr = c_dma; 2231 2232 /* Fill microcode part of instruction */ 2233 req->ist.ei0 = vq_cmd_w0.u64; 2234 2235 req->op = op; 2236 2237 *prep_req = req; 2238 return; 2239 } 2240 2241 static __rte_always_inline void 2242 cpt_kasumi_dec_prep(uint64_t d_offs, 2243 uint64_t d_lens, 2244 fc_params_t *params, 2245 void *op, 2246 void **prep_req) 2247 { 2248 uint32_t size; 2249 int32_t inputlen = 0, outputlen; 2250 struct cpt_ctx *cpt_ctx; 2251 uint8_t i = 0, iv_len = 8; 2252 struct cpt_request_info *req; 2253 buf_ptr_t *buf_p; 2254 uint32_t encr_offset; 2255 uint32_t encr_data_len; 2256 int flags; 2257 uint8_t dir = 0; 2258 void *m_vaddr, *c_vaddr; 2259 uint64_t m_dma, c_dma; 2260 uint64_t *offset_vaddr, offset_dma; 2261 vq_cmd_word0_t vq_cmd_w0; 2262 uint8_t *in_buffer; 2263 uint32_t g_size_bytes, s_size_bytes; 2264 uint64_t dptr_dma, rptr_dma; 2265 sg_comp_t *gather_comp; 2266 sg_comp_t *scatter_comp; 2267 2268 buf_p = ¶ms->meta_buf; 2269 m_vaddr = buf_p->vaddr; 2270 m_dma = buf_p->dma_addr; 2271 2272 encr_offset = ENCR_OFFSET(d_offs) / 8; 2273 encr_data_len = ENCR_DLEN(d_lens); 2274 2275 cpt_ctx = params->ctx_buf.vaddr; 2276 flags = cpt_ctx->zsk_flags; 2277 /* 2278 * Save initial space that followed app data for completion code & 2279 * alternate completion code to fall in same cache line as app data 2280 */ 2281 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2282 m_dma += COMPLETION_CODE_SIZE; 2283 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2284 (uint8_t *)m_vaddr; 2285 2286 c_vaddr = (uint8_t *)m_vaddr + size; 2287 c_dma = m_dma + size; 2288 size += sizeof(cpt_res_s_t); 2289 2290 m_vaddr = (uint8_t *)m_vaddr + size; 2291 m_dma += size; 2292 2293 /* Reserve memory for cpt request info */ 2294 req = m_vaddr; 2295 2296 size = sizeof(struct cpt_request_info); 2297 m_vaddr = (uint8_t *)m_vaddr + size; 2298 m_dma += size; 2299 2300 vq_cmd_w0.u64 = 0; 2301 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2302 2303 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2304 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2305 (dir << 4) | (0 << 3) | (flags & 0x7)); 2306 2307 /* 2308 * GP op header, lengths are expected in bits. 2309 */ 2310 vq_cmd_w0.s.param1 = encr_data_len; 2311 2312 /* consider iv len */ 2313 encr_offset += iv_len; 2314 2315 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2316 outputlen = inputlen; 2317 2318 /* save space for offset ctrl & iv */ 2319 offset_vaddr = m_vaddr; 2320 offset_dma = m_dma; 2321 2322 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2323 m_dma += OFF_CTRL_LEN + iv_len; 2324 2325 /* DPTR has SG list */ 2326 in_buffer = m_vaddr; 2327 dptr_dma = m_dma; 2328 2329 ((uint16_t *)in_buffer)[0] = 0; 2330 ((uint16_t *)in_buffer)[1] = 0; 2331 2332 /* TODO Add error check if space will be sufficient */ 2333 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2334 2335 /* 2336 * Input Gather List 2337 */ 2338 i = 0; 2339 2340 /* Offset control word followed by iv */ 2341 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2342 if (unlikely((encr_offset >> 16))) { 2343 CPT_LOG_DP_ERR("Offset not supported"); 2344 CPT_LOG_DP_ERR("enc_offset: %d", encr_offset); 2345 return; 2346 } 2347 2348 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2349 2350 /* IV */ 2351 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2352 params->iv_buf, iv_len); 2353 2354 /* Add input data */ 2355 size = inputlen - iv_len; 2356 if (size) { 2357 i = fill_sg_comp_from_iov(gather_comp, i, 2358 params->src_iov, 2359 0, &size, NULL, 0); 2360 if (unlikely(size)) { 2361 CPT_LOG_DP_ERR("Insufficient buffer space," 2362 " size %d needed", size); 2363 return; 2364 } 2365 } 2366 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2367 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2368 2369 /* 2370 * Output Scatter List 2371 */ 2372 2373 i = 0; 2374 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2375 2376 /* IV */ 2377 i = fill_sg_comp(scatter_comp, i, 2378 offset_dma + OFF_CTRL_LEN, 2379 iv_len); 2380 2381 /* Add output data */ 2382 size = outputlen - iv_len; 2383 if (size) { 2384 i = fill_sg_comp_from_iov(scatter_comp, i, 2385 params->dst_iov, 0, 2386 &size, NULL, 0); 2387 if (unlikely(size)) { 2388 CPT_LOG_DP_ERR("Insufficient buffer space," 2389 " size %d needed", size); 2390 return; 2391 } 2392 } 2393 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2394 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2395 2396 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2397 2398 /* This is DPTR len incase of SG mode */ 2399 vq_cmd_w0.s.dlen = size; 2400 2401 m_vaddr = (uint8_t *)m_vaddr + size; 2402 m_dma += size; 2403 2404 /* cpt alternate completion address saved earlier */ 2405 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2406 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2407 rptr_dma = c_dma - 8; 2408 2409 req->ist.ei1 = dptr_dma; 2410 req->ist.ei2 = rptr_dma; 2411 2412 /* 16 byte aligned cpt res address */ 2413 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2414 *req->completion_addr = COMPLETION_CODE_INIT; 2415 req->comp_baddr = c_dma; 2416 2417 /* Fill microcode part of instruction */ 2418 req->ist.ei0 = vq_cmd_w0.u64; 2419 2420 req->op = op; 2421 2422 *prep_req = req; 2423 return; 2424 } 2425 2426 static __rte_always_inline void * 2427 cpt_fc_dec_hmac_prep(uint32_t flags, 2428 uint64_t d_offs, 2429 uint64_t d_lens, 2430 fc_params_t *fc_params, 2431 void *op) 2432 { 2433 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2434 uint8_t fc_type; 2435 void *prep_req = NULL; 2436 2437 fc_type = ctx->fc_type; 2438 2439 if (likely(fc_type == FC_GEN)) { 2440 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2441 &prep_req); 2442 } else if (fc_type == ZUC_SNOW3G) { 2443 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2444 &prep_req); 2445 } else if (fc_type == KASUMI) { 2446 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2447 } 2448 2449 /* 2450 * For AUTH_ONLY case, 2451 * MC only supports digest generation and verification 2452 * should be done in software by memcmp() 2453 */ 2454 2455 return prep_req; 2456 } 2457 2458 static __rte_always_inline void *__rte_hot 2459 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2460 fc_params_t *fc_params, void *op) 2461 { 2462 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2463 uint8_t fc_type; 2464 void *prep_req = NULL; 2465 2466 fc_type = ctx->fc_type; 2467 2468 /* Common api for rest of the ops */ 2469 if (likely(fc_type == FC_GEN)) { 2470 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2471 &prep_req); 2472 } else if (fc_type == ZUC_SNOW3G) { 2473 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2474 &prep_req); 2475 } else if (fc_type == KASUMI) { 2476 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2477 &prep_req); 2478 } else if (fc_type == HASH_HMAC) { 2479 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2480 } 2481 2482 return prep_req; 2483 } 2484 2485 static __rte_always_inline int 2486 cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type, 2487 const uint8_t *key, uint16_t key_len, uint16_t mac_len) 2488 { 2489 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 2490 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 2491 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 2492 2493 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2494 uint32_t keyx[4]; 2495 2496 if (key_len != 16) 2497 return -1; 2498 /* No support for AEAD yet */ 2499 if (cpt_ctx->enc_cipher) 2500 return -1; 2501 /* For ZUC/SNOW3G/Kasumi */ 2502 switch (type) { 2503 case SNOW3G_UIA2: 2504 cpt_ctx->snow3g = 1; 2505 gen_key_snow3g(key, keyx); 2506 memcpy(zs_ctx->ci_key, keyx, key_len); 2507 cpt_ctx->fc_type = ZUC_SNOW3G; 2508 cpt_ctx->zsk_flags = 0x1; 2509 break; 2510 case ZUC_EIA3: 2511 cpt_ctx->snow3g = 0; 2512 memcpy(zs_ctx->ci_key, key, key_len); 2513 memcpy(zs_ctx->zuc_const, zuc_d, 32); 2514 cpt_ctx->fc_type = ZUC_SNOW3G; 2515 cpt_ctx->zsk_flags = 0x1; 2516 break; 2517 case KASUMI_F9_ECB: 2518 /* Kasumi ECB mode */ 2519 cpt_ctx->k_ecb = 1; 2520 memcpy(k_ctx->ci_key, key, key_len); 2521 cpt_ctx->fc_type = KASUMI; 2522 cpt_ctx->zsk_flags = 0x1; 2523 break; 2524 case KASUMI_F9_CBC: 2525 memcpy(k_ctx->ci_key, key, key_len); 2526 cpt_ctx->fc_type = KASUMI; 2527 cpt_ctx->zsk_flags = 0x1; 2528 break; 2529 default: 2530 return -1; 2531 } 2532 cpt_ctx->mac_len = 4; 2533 cpt_ctx->hash_type = type; 2534 return 0; 2535 } 2536 2537 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2538 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2539 cpt_ctx->fc_type = HASH_HMAC; 2540 } 2541 2542 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2543 return -1; 2544 2545 /* For GMAC auth, cipher must be NULL */ 2546 if (type == GMAC_TYPE) 2547 fctx->enc.enc_cipher = 0; 2548 2549 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2550 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2551 2552 if (key_len) { 2553 cpt_ctx->hmac = 1; 2554 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2555 memcpy(cpt_ctx->auth_key, key, key_len); 2556 cpt_ctx->auth_key_len = key_len; 2557 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2558 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2559 2560 if (key_len <= 64) 2561 memcpy(fctx->hmac.opad, key, key_len); 2562 fctx->enc.auth_input_type = 1; 2563 } 2564 return 0; 2565 } 2566 2567 static __rte_always_inline int 2568 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2569 struct cpt_sess_misc *sess) 2570 { 2571 struct rte_crypto_aead_xform *aead_form; 2572 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2573 auth_type_t auth_type = 0; /* NULL Auth type */ 2574 uint32_t cipher_key_len = 0; 2575 uint8_t aes_gcm = 0; 2576 aead_form = &xform->aead; 2577 void *ctx = SESS_PRIV(sess); 2578 2579 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 2580 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2581 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2582 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) { 2583 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2584 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2585 } else { 2586 CPT_LOG_DP_ERR("Unknown aead operation\n"); 2587 return -1; 2588 } 2589 switch (aead_form->algo) { 2590 case RTE_CRYPTO_AEAD_AES_GCM: 2591 enc_type = AES_GCM; 2592 cipher_key_len = 16; 2593 aes_gcm = 1; 2594 break; 2595 case RTE_CRYPTO_AEAD_AES_CCM: 2596 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2597 aead_form->algo); 2598 return -1; 2599 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: 2600 enc_type = CHACHA20; 2601 auth_type = POLY1305; 2602 cipher_key_len = 32; 2603 sess->chacha_poly = 1; 2604 break; 2605 default: 2606 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2607 aead_form->algo); 2608 return -1; 2609 } 2610 if (aead_form->key.length < cipher_key_len) { 2611 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2612 (unsigned int long)aead_form->key.length); 2613 return -1; 2614 } 2615 sess->zsk_flag = 0; 2616 sess->aes_gcm = aes_gcm; 2617 sess->mac_len = aead_form->digest_length; 2618 sess->iv_offset = aead_form->iv.offset; 2619 sess->iv_length = aead_form->iv.length; 2620 sess->aad_length = aead_form->aad_length; 2621 2622 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2623 aead_form->key.length, NULL))) 2624 return -1; 2625 2626 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2627 aead_form->digest_length))) 2628 return -1; 2629 2630 return 0; 2631 } 2632 2633 static __rte_always_inline int 2634 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2635 struct cpt_sess_misc *sess) 2636 { 2637 struct rte_crypto_cipher_xform *c_form; 2638 struct cpt_ctx *ctx = SESS_PRIV(sess); 2639 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2640 uint32_t cipher_key_len = 0; 2641 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2642 2643 c_form = &xform->cipher; 2644 2645 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2646 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2647 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 2648 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2649 if (xform->next != NULL && 2650 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2651 /* Perform decryption followed by auth verify */ 2652 ctx->dec_auth = 1; 2653 } 2654 } else { 2655 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2656 return -1; 2657 } 2658 2659 switch (c_form->algo) { 2660 case RTE_CRYPTO_CIPHER_AES_CBC: 2661 enc_type = AES_CBC; 2662 cipher_key_len = 16; 2663 break; 2664 case RTE_CRYPTO_CIPHER_3DES_CBC: 2665 enc_type = DES3_CBC; 2666 cipher_key_len = 24; 2667 break; 2668 case RTE_CRYPTO_CIPHER_DES_CBC: 2669 /* DES is implemented using 3DES in hardware */ 2670 enc_type = DES3_CBC; 2671 cipher_key_len = 8; 2672 break; 2673 case RTE_CRYPTO_CIPHER_AES_CTR: 2674 enc_type = AES_CTR; 2675 cipher_key_len = 16; 2676 aes_ctr = 1; 2677 break; 2678 case RTE_CRYPTO_CIPHER_NULL: 2679 enc_type = 0; 2680 is_null = 1; 2681 break; 2682 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2683 enc_type = KASUMI_F8_ECB; 2684 cipher_key_len = 16; 2685 zsk_flag = K_F8; 2686 break; 2687 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2688 enc_type = SNOW3G_UEA2; 2689 cipher_key_len = 16; 2690 zsk_flag = ZS_EA; 2691 break; 2692 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2693 enc_type = ZUC_EEA3; 2694 cipher_key_len = 16; 2695 zsk_flag = ZS_EA; 2696 break; 2697 case RTE_CRYPTO_CIPHER_AES_XTS: 2698 enc_type = AES_XTS; 2699 cipher_key_len = 16; 2700 break; 2701 case RTE_CRYPTO_CIPHER_3DES_ECB: 2702 enc_type = DES3_ECB; 2703 cipher_key_len = 24; 2704 break; 2705 case RTE_CRYPTO_CIPHER_AES_ECB: 2706 enc_type = AES_ECB; 2707 cipher_key_len = 16; 2708 break; 2709 case RTE_CRYPTO_CIPHER_3DES_CTR: 2710 case RTE_CRYPTO_CIPHER_AES_F8: 2711 case RTE_CRYPTO_CIPHER_ARC4: 2712 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2713 c_form->algo); 2714 return -1; 2715 default: 2716 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2717 c_form->algo); 2718 return -1; 2719 } 2720 2721 if (c_form->key.length < cipher_key_len) { 2722 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2723 (unsigned long) c_form->key.length); 2724 return -1; 2725 } 2726 2727 sess->zsk_flag = zsk_flag; 2728 sess->aes_gcm = 0; 2729 sess->aes_ctr = aes_ctr; 2730 sess->iv_offset = c_form->iv.offset; 2731 sess->iv_length = c_form->iv.length; 2732 sess->is_null = is_null; 2733 2734 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2735 c_form->key.data, c_form->key.length, NULL))) 2736 return -1; 2737 2738 return 0; 2739 } 2740 2741 static __rte_always_inline int 2742 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2743 struct cpt_sess_misc *sess) 2744 { 2745 struct cpt_ctx *ctx = SESS_PRIV(sess); 2746 struct rte_crypto_auth_xform *a_form; 2747 auth_type_t auth_type = 0; /* NULL Auth type */ 2748 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2749 2750 if (xform->next != NULL && 2751 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2752 xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 2753 /* Perform auth followed by encryption */ 2754 ctx->auth_enc = 1; 2755 } 2756 2757 a_form = &xform->auth; 2758 2759 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2760 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2761 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2762 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2763 else { 2764 CPT_LOG_DP_ERR("Unknown auth operation"); 2765 return -1; 2766 } 2767 2768 switch (a_form->algo) { 2769 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2770 /* Fall through */ 2771 case RTE_CRYPTO_AUTH_SHA1: 2772 auth_type = SHA1_TYPE; 2773 break; 2774 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2775 case RTE_CRYPTO_AUTH_SHA256: 2776 auth_type = SHA2_SHA256; 2777 break; 2778 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2779 case RTE_CRYPTO_AUTH_SHA512: 2780 auth_type = SHA2_SHA512; 2781 break; 2782 case RTE_CRYPTO_AUTH_AES_GMAC: 2783 auth_type = GMAC_TYPE; 2784 aes_gcm = 1; 2785 break; 2786 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2787 case RTE_CRYPTO_AUTH_SHA224: 2788 auth_type = SHA2_SHA224; 2789 break; 2790 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2791 case RTE_CRYPTO_AUTH_SHA384: 2792 auth_type = SHA2_SHA384; 2793 break; 2794 case RTE_CRYPTO_AUTH_MD5_HMAC: 2795 case RTE_CRYPTO_AUTH_MD5: 2796 auth_type = MD5_TYPE; 2797 break; 2798 case RTE_CRYPTO_AUTH_KASUMI_F9: 2799 auth_type = KASUMI_F9_ECB; 2800 /* 2801 * Indicate that direction needs to be taken out 2802 * from end of src 2803 */ 2804 zsk_flag = K_F9; 2805 break; 2806 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2807 auth_type = SNOW3G_UIA2; 2808 zsk_flag = ZS_IA; 2809 break; 2810 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2811 auth_type = ZUC_EIA3; 2812 zsk_flag = ZS_IA; 2813 break; 2814 case RTE_CRYPTO_AUTH_NULL: 2815 auth_type = 0; 2816 is_null = 1; 2817 break; 2818 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2819 case RTE_CRYPTO_AUTH_AES_CMAC: 2820 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2821 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2822 a_form->algo); 2823 return -1; 2824 default: 2825 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2826 a_form->algo); 2827 return -1; 2828 } 2829 2830 sess->zsk_flag = zsk_flag; 2831 sess->aes_gcm = aes_gcm; 2832 sess->mac_len = a_form->digest_length; 2833 sess->is_null = is_null; 2834 if (zsk_flag) { 2835 sess->auth_iv_offset = a_form->iv.offset; 2836 sess->auth_iv_length = a_form->iv.length; 2837 } 2838 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2839 a_form->key.data, a_form->key.length, 2840 a_form->digest_length))) 2841 return -1; 2842 2843 return 0; 2844 } 2845 2846 static __rte_always_inline int 2847 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2848 struct cpt_sess_misc *sess) 2849 { 2850 struct rte_crypto_auth_xform *a_form; 2851 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2852 auth_type_t auth_type = 0; /* NULL Auth type */ 2853 void *ctx = SESS_PRIV(sess); 2854 2855 a_form = &xform->auth; 2856 2857 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2858 sess->cpt_op |= CPT_OP_ENCODE; 2859 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2860 sess->cpt_op |= CPT_OP_DECODE; 2861 else { 2862 CPT_LOG_DP_ERR("Unknown auth operation"); 2863 return -1; 2864 } 2865 2866 switch (a_form->algo) { 2867 case RTE_CRYPTO_AUTH_AES_GMAC: 2868 enc_type = AES_GCM; 2869 auth_type = GMAC_TYPE; 2870 break; 2871 default: 2872 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2873 a_form->algo); 2874 return -1; 2875 } 2876 2877 sess->zsk_flag = 0; 2878 sess->aes_gcm = 0; 2879 sess->is_gmac = 1; 2880 sess->iv_offset = a_form->iv.offset; 2881 sess->iv_length = a_form->iv.length; 2882 sess->mac_len = a_form->digest_length; 2883 2884 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2885 a_form->key.length, NULL))) 2886 return -1; 2887 2888 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2889 a_form->digest_length))) 2890 return -1; 2891 2892 return 0; 2893 } 2894 2895 static __rte_always_inline void * 2896 alloc_op_meta(struct rte_mbuf *m_src, 2897 buf_ptr_t *buf, 2898 int32_t len, 2899 struct rte_mempool *cpt_meta_pool) 2900 { 2901 uint8_t *mdata; 2902 2903 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2904 if (likely(m_src && (m_src->nb_segs == 1))) { 2905 int32_t tailroom; 2906 phys_addr_t mphys; 2907 2908 /* Check if tailroom is sufficient to hold meta data */ 2909 tailroom = rte_pktmbuf_tailroom(m_src); 2910 if (likely(tailroom > len + 8)) { 2911 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2912 mphys = m_src->buf_iova + m_src->buf_len; 2913 mdata -= len; 2914 mphys -= len; 2915 buf->vaddr = mdata; 2916 buf->dma_addr = mphys; 2917 buf->size = len; 2918 /* Indicate that this is a mbuf allocated mdata */ 2919 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2920 return mdata; 2921 } 2922 } 2923 #else 2924 RTE_SET_USED(m_src); 2925 #endif 2926 2927 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2928 return NULL; 2929 2930 buf->vaddr = mdata; 2931 buf->dma_addr = rte_mempool_virt2iova(mdata); 2932 buf->size = len; 2933 2934 return mdata; 2935 } 2936 2937 /** 2938 * cpt_free_metabuf - free metabuf to mempool. 2939 * @param instance: pointer to instance. 2940 * @param objp: pointer to the metabuf. 2941 */ 2942 static __rte_always_inline void 2943 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2944 { 2945 bool nofree = ((uintptr_t)mdata & 1ull); 2946 2947 if (likely(nofree)) 2948 return; 2949 rte_mempool_put(cpt_meta_pool, mdata); 2950 } 2951 2952 static __rte_always_inline uint32_t 2953 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2954 iov_ptr_t *iovec, uint32_t start_offset) 2955 { 2956 uint16_t index = 0; 2957 void *seg_data = NULL; 2958 phys_addr_t seg_phys; 2959 int32_t seg_size = 0; 2960 2961 if (!pkt) { 2962 iovec->buf_cnt = 0; 2963 return 0; 2964 } 2965 2966 if (!start_offset) { 2967 seg_data = rte_pktmbuf_mtod(pkt, void *); 2968 seg_phys = rte_pktmbuf_iova(pkt); 2969 seg_size = pkt->data_len; 2970 } else { 2971 while (start_offset >= pkt->data_len) { 2972 start_offset -= pkt->data_len; 2973 pkt = pkt->next; 2974 } 2975 2976 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2977 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset); 2978 seg_size = pkt->data_len - start_offset; 2979 if (!seg_size) 2980 return 1; 2981 } 2982 2983 /* first seg */ 2984 iovec->bufs[index].vaddr = seg_data; 2985 iovec->bufs[index].dma_addr = seg_phys; 2986 iovec->bufs[index].size = seg_size; 2987 index++; 2988 pkt = pkt->next; 2989 2990 while (unlikely(pkt != NULL)) { 2991 seg_data = rte_pktmbuf_mtod(pkt, void *); 2992 seg_phys = rte_pktmbuf_iova(pkt); 2993 seg_size = pkt->data_len; 2994 if (!seg_size) 2995 break; 2996 2997 iovec->bufs[index].vaddr = seg_data; 2998 iovec->bufs[index].dma_addr = seg_phys; 2999 iovec->bufs[index].size = seg_size; 3000 3001 index++; 3002 3003 pkt = pkt->next; 3004 } 3005 3006 iovec->buf_cnt = index; 3007 return 0; 3008 } 3009 3010 static __rte_always_inline uint32_t 3011 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 3012 fc_params_t *param, 3013 uint32_t *flags) 3014 { 3015 uint16_t index = 0; 3016 void *seg_data = NULL; 3017 phys_addr_t seg_phys; 3018 uint32_t seg_size = 0; 3019 iov_ptr_t *iovec; 3020 3021 seg_data = rte_pktmbuf_mtod(pkt, void *); 3022 seg_phys = rte_pktmbuf_iova(pkt); 3023 seg_size = pkt->data_len; 3024 3025 /* first seg */ 3026 if (likely(!pkt->next)) { 3027 uint32_t headroom, tailroom; 3028 3029 *flags |= SINGLE_BUF_INPLACE; 3030 headroom = rte_pktmbuf_headroom(pkt); 3031 tailroom = rte_pktmbuf_tailroom(pkt); 3032 if (likely((headroom >= 24) && 3033 (tailroom >= 8))) { 3034 /* In 83XX this is prerequivisit for Direct mode */ 3035 *flags |= SINGLE_BUF_HEADTAILROOM; 3036 } 3037 param->bufs[0].vaddr = seg_data; 3038 param->bufs[0].dma_addr = seg_phys; 3039 param->bufs[0].size = seg_size; 3040 return 0; 3041 } 3042 iovec = param->src_iov; 3043 iovec->bufs[index].vaddr = seg_data; 3044 iovec->bufs[index].dma_addr = seg_phys; 3045 iovec->bufs[index].size = seg_size; 3046 index++; 3047 pkt = pkt->next; 3048 3049 while (unlikely(pkt != NULL)) { 3050 seg_data = rte_pktmbuf_mtod(pkt, void *); 3051 seg_phys = rte_pktmbuf_iova(pkt); 3052 seg_size = pkt->data_len; 3053 3054 if (!seg_size) 3055 break; 3056 3057 iovec->bufs[index].vaddr = seg_data; 3058 iovec->bufs[index].dma_addr = seg_phys; 3059 iovec->bufs[index].size = seg_size; 3060 3061 index++; 3062 3063 pkt = pkt->next; 3064 } 3065 3066 iovec->buf_cnt = index; 3067 return 0; 3068 } 3069 3070 static __rte_always_inline int 3071 fill_fc_params(struct rte_crypto_op *cop, 3072 struct cpt_sess_misc *sess_misc, 3073 struct cpt_qp_meta_info *m_info, 3074 void **mdata_ptr, 3075 void **prep_req) 3076 { 3077 uint32_t space = 0; 3078 struct rte_crypto_sym_op *sym_op = cop->sym; 3079 struct cpt_ctx *ctx = SESS_PRIV(sess_misc); 3080 void *mdata = NULL; 3081 uintptr_t *op; 3082 uint32_t mc_hash_off; 3083 uint32_t flags = 0; 3084 uint64_t d_offs, d_lens; 3085 struct rte_mbuf *m_src, *m_dst; 3086 uint8_t cpt_op = sess_misc->cpt_op; 3087 #ifdef CPT_ALWAYS_USE_SG_MODE 3088 uint8_t inplace = 0; 3089 #else 3090 uint8_t inplace = 1; 3091 #endif 3092 fc_params_t fc_params; 3093 char src[SRC_IOV_SIZE]; 3094 char dst[SRC_IOV_SIZE]; 3095 uint32_t iv_buf[4]; 3096 int ret; 3097 3098 if (likely(sess_misc->iv_length)) { 3099 flags |= VALID_IV_BUF; 3100 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3101 uint8_t *, sess_misc->iv_offset); 3102 if (sess_misc->aes_ctr && 3103 unlikely(sess_misc->iv_length != 16)) { 3104 memcpy((uint8_t *)iv_buf, 3105 rte_crypto_op_ctod_offset(cop, 3106 uint8_t *, sess_misc->iv_offset), 12); 3107 iv_buf[3] = rte_cpu_to_be_32(0x1); 3108 fc_params.iv_buf = iv_buf; 3109 } 3110 } 3111 3112 if (sess_misc->zsk_flag) { 3113 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3114 uint8_t *, 3115 sess_misc->auth_iv_offset); 3116 if (sess_misc->zsk_flag != ZS_EA) 3117 inplace = 0; 3118 } 3119 m_src = sym_op->m_src; 3120 m_dst = sym_op->m_dst; 3121 3122 if (sess_misc->aes_gcm || sess_misc->chacha_poly) { 3123 uint8_t *salt; 3124 uint8_t *aad_data; 3125 uint16_t aad_len; 3126 3127 d_offs = sym_op->aead.data.offset; 3128 d_lens = sym_op->aead.data.length; 3129 mc_hash_off = sym_op->aead.data.offset + 3130 sym_op->aead.data.length; 3131 3132 aad_data = sym_op->aead.aad.data; 3133 aad_len = sess_misc->aad_length; 3134 if (likely((aad_data + aad_len) == 3135 rte_pktmbuf_mtod_offset(m_src, 3136 uint8_t *, 3137 sym_op->aead.data.offset))) { 3138 d_offs = (d_offs - aad_len) | (d_offs << 16); 3139 d_lens = (d_lens + aad_len) | (d_lens << 32); 3140 } else { 3141 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3142 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3143 fc_params.aad_buf.size = aad_len; 3144 flags |= VALID_AAD_BUF; 3145 inplace = 0; 3146 d_offs = d_offs << 16; 3147 d_lens = d_lens << 32; 3148 } 3149 3150 salt = fc_params.iv_buf; 3151 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3152 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3153 sess_misc->salt = *(uint32_t *)salt; 3154 } 3155 fc_params.iv_buf = salt + 4; 3156 if (likely(sess_misc->mac_len)) { 3157 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3158 m_src; 3159 3160 if (!m) 3161 m = m_src; 3162 3163 /* hmac immediately following data is best case */ 3164 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3165 mc_hash_off != 3166 (uint8_t *)sym_op->aead.digest.data)) { 3167 flags |= VALID_MAC_BUF; 3168 fc_params.mac_buf.size = sess_misc->mac_len; 3169 fc_params.mac_buf.vaddr = 3170 sym_op->aead.digest.data; 3171 fc_params.mac_buf.dma_addr = 3172 sym_op->aead.digest.phys_addr; 3173 inplace = 0; 3174 } 3175 } 3176 } else { 3177 d_offs = sym_op->cipher.data.offset; 3178 d_lens = sym_op->cipher.data.length; 3179 mc_hash_off = sym_op->cipher.data.offset + 3180 sym_op->cipher.data.length; 3181 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3182 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3183 3184 if (mc_hash_off < (sym_op->auth.data.offset + 3185 sym_op->auth.data.length)){ 3186 mc_hash_off = (sym_op->auth.data.offset + 3187 sym_op->auth.data.length); 3188 } 3189 /* for gmac, salt should be updated like in gcm */ 3190 if (unlikely(sess_misc->is_gmac)) { 3191 uint8_t *salt; 3192 salt = fc_params.iv_buf; 3193 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3194 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3195 sess_misc->salt = *(uint32_t *)salt; 3196 } 3197 fc_params.iv_buf = salt + 4; 3198 } 3199 if (likely(sess_misc->mac_len)) { 3200 struct rte_mbuf *m; 3201 3202 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3203 if (!m) 3204 m = m_src; 3205 3206 /* hmac immediately following data is best case */ 3207 if (!ctx->dec_auth && !ctx->auth_enc && 3208 (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3209 mc_hash_off != 3210 (uint8_t *)sym_op->auth.digest.data))) { 3211 flags |= VALID_MAC_BUF; 3212 fc_params.mac_buf.size = 3213 sess_misc->mac_len; 3214 fc_params.mac_buf.vaddr = 3215 sym_op->auth.digest.data; 3216 fc_params.mac_buf.dma_addr = 3217 sym_op->auth.digest.phys_addr; 3218 inplace = 0; 3219 } 3220 } 3221 } 3222 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3223 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3224 3225 if (!ctx->dec_auth && 3226 unlikely(sess_misc->is_null || 3227 sess_misc->cpt_op == CPT_OP_DECODE)) 3228 inplace = 0; 3229 3230 if (likely(!m_dst && inplace)) { 3231 /* Case of single buffer without AAD buf or 3232 * separate mac buf in place and 3233 * not air crypto 3234 */ 3235 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3236 3237 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3238 &fc_params, 3239 &flags))) { 3240 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3241 ret = -EINVAL; 3242 goto err_exit; 3243 } 3244 3245 } else { 3246 /* Out of place processing */ 3247 fc_params.src_iov = (void *)src; 3248 fc_params.dst_iov = (void *)dst; 3249 3250 /* Store SG I/O in the api for reuse */ 3251 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3252 CPT_LOG_DP_ERR("Prepare src iov failed"); 3253 ret = -EINVAL; 3254 goto err_exit; 3255 } 3256 3257 if (unlikely(m_dst != NULL)) { 3258 uint32_t pkt_len; 3259 3260 /* Try to make room as much as src has */ 3261 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3262 3263 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3264 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3265 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3266 CPT_LOG_DP_ERR("Not enough space in " 3267 "m_dst %p, need %u" 3268 " more", 3269 m_dst, pkt_len); 3270 ret = -EINVAL; 3271 goto err_exit; 3272 } 3273 } 3274 3275 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3276 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3277 "m_dst %p", m_dst); 3278 ret = -EINVAL; 3279 goto err_exit; 3280 } 3281 } else { 3282 fc_params.dst_iov = (void *)src; 3283 } 3284 } 3285 3286 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3287 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3288 m_info->lb_mlen, m_info->pool); 3289 else 3290 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3291 m_info->sg_mlen, m_info->pool); 3292 3293 if (unlikely(mdata == NULL)) { 3294 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3295 ret = -ENOMEM; 3296 goto err_exit; 3297 } 3298 3299 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3300 op[0] = (uintptr_t)mdata; 3301 op[1] = (uintptr_t)cop; 3302 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3303 space += 4 * sizeof(uint64_t); 3304 3305 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3306 fc_params.meta_buf.dma_addr += space; 3307 fc_params.meta_buf.size -= space; 3308 3309 /* Finally prepare the instruction */ 3310 if (cpt_op & CPT_OP_ENCODE) 3311 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3312 &fc_params, op); 3313 else 3314 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3315 &fc_params, op); 3316 3317 if (unlikely(*prep_req == NULL)) { 3318 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3319 ret = -EINVAL; 3320 goto free_mdata_and_exit; 3321 } 3322 3323 *mdata_ptr = mdata; 3324 3325 return 0; 3326 3327 free_mdata_and_exit: 3328 free_op_meta(mdata, m_info->pool); 3329 err_exit: 3330 return ret; 3331 } 3332 3333 static __rte_always_inline void 3334 compl_auth_verify(struct rte_crypto_op *op, 3335 uint8_t *gen_mac, 3336 uint64_t mac_len) 3337 { 3338 uint8_t *mac; 3339 struct rte_crypto_sym_op *sym_op = op->sym; 3340 3341 if (sym_op->auth.digest.data) 3342 mac = sym_op->auth.digest.data; 3343 else 3344 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3345 uint8_t *, 3346 sym_op->auth.data.length + 3347 sym_op->auth.data.offset); 3348 if (!mac) { 3349 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3350 return; 3351 } 3352 3353 if (memcmp(mac, gen_mac, mac_len)) 3354 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3355 else 3356 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3357 } 3358 3359 static __rte_always_inline void 3360 find_kasumif9_direction_and_length(uint8_t *src, 3361 uint32_t counter_num_bytes, 3362 uint32_t *addr_length_in_bits, 3363 uint8_t *addr_direction) 3364 { 3365 uint8_t found = 0; 3366 uint32_t pos; 3367 uint8_t last_byte; 3368 while (!found && counter_num_bytes > 0) { 3369 counter_num_bytes--; 3370 if (src[counter_num_bytes] == 0x00) 3371 continue; 3372 pos = rte_bsf32(src[counter_num_bytes]); 3373 if (pos == 7) { 3374 if (likely(counter_num_bytes > 0)) { 3375 last_byte = src[counter_num_bytes - 1]; 3376 *addr_direction = last_byte & 0x1; 3377 *addr_length_in_bits = counter_num_bytes * 8 3378 - 1; 3379 } 3380 } else { 3381 last_byte = src[counter_num_bytes]; 3382 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3383 *addr_length_in_bits = counter_num_bytes * 8 3384 + (8 - (pos + 2)); 3385 } 3386 found = 1; 3387 } 3388 } 3389 3390 /* 3391 * This handles all auth only except AES_GMAC 3392 */ 3393 static __rte_always_inline int 3394 fill_digest_params(struct rte_crypto_op *cop, 3395 struct cpt_sess_misc *sess, 3396 struct cpt_qp_meta_info *m_info, 3397 void **mdata_ptr, 3398 void **prep_req) 3399 { 3400 uint32_t space = 0; 3401 struct rte_crypto_sym_op *sym_op = cop->sym; 3402 void *mdata; 3403 phys_addr_t mphys; 3404 uint64_t *op; 3405 uint32_t auth_range_off; 3406 uint32_t flags = 0; 3407 uint64_t d_offs = 0, d_lens; 3408 struct rte_mbuf *m_src, *m_dst; 3409 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3410 uint16_t mac_len = sess->mac_len; 3411 fc_params_t params; 3412 char src[SRC_IOV_SIZE]; 3413 uint8_t iv_buf[16]; 3414 int ret; 3415 3416 memset(¶ms, 0, sizeof(fc_params_t)); 3417 3418 m_src = sym_op->m_src; 3419 3420 /* For just digest lets force mempool alloc */ 3421 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3422 m_info->pool); 3423 if (mdata == NULL) { 3424 ret = -ENOMEM; 3425 goto err_exit; 3426 } 3427 3428 mphys = params.meta_buf.dma_addr; 3429 3430 op = mdata; 3431 op[0] = (uintptr_t)mdata; 3432 op[1] = (uintptr_t)cop; 3433 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3434 space += 4 * sizeof(uint64_t); 3435 3436 auth_range_off = sym_op->auth.data.offset; 3437 3438 flags = VALID_MAC_BUF; 3439 params.src_iov = (void *)src; 3440 if (unlikely(sess->zsk_flag)) { 3441 /* 3442 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3443 * we will send pass through even for auth only case, 3444 * let MC handle it 3445 */ 3446 d_offs = auth_range_off; 3447 auth_range_off = 0; 3448 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3449 uint8_t *, sess->auth_iv_offset); 3450 if (sess->zsk_flag == K_F9) { 3451 uint32_t length_in_bits, num_bytes; 3452 uint8_t *src, direction = 0; 3453 3454 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3455 uint8_t *), 8); 3456 /* 3457 * This is kasumi f9, take direction from 3458 * source buffer 3459 */ 3460 length_in_bits = cop->sym->auth.data.length; 3461 num_bytes = (length_in_bits >> 3); 3462 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3463 find_kasumif9_direction_and_length(src, 3464 num_bytes, 3465 &length_in_bits, 3466 &direction); 3467 length_in_bits -= 64; 3468 cop->sym->auth.data.offset += 64; 3469 d_offs = cop->sym->auth.data.offset; 3470 auth_range_off = d_offs / 8; 3471 cop->sym->auth.data.length = length_in_bits; 3472 3473 /* Store it at end of auth iv */ 3474 iv_buf[8] = direction; 3475 params.auth_iv_buf = iv_buf; 3476 } 3477 } 3478 3479 d_lens = sym_op->auth.data.length; 3480 3481 params.ctx_buf.vaddr = SESS_PRIV(sess); 3482 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3483 3484 if (auth_op == CPT_OP_AUTH_GENERATE) { 3485 if (sym_op->auth.digest.data) { 3486 /* 3487 * Digest to be generated 3488 * in separate buffer 3489 */ 3490 params.mac_buf.size = 3491 sess->mac_len; 3492 params.mac_buf.vaddr = 3493 sym_op->auth.digest.data; 3494 params.mac_buf.dma_addr = 3495 sym_op->auth.digest.phys_addr; 3496 } else { 3497 uint32_t off = sym_op->auth.data.offset + 3498 sym_op->auth.data.length; 3499 int32_t dlen, space; 3500 3501 m_dst = sym_op->m_dst ? 3502 sym_op->m_dst : sym_op->m_src; 3503 dlen = rte_pktmbuf_pkt_len(m_dst); 3504 3505 space = off + mac_len - dlen; 3506 if (space > 0) 3507 if (!rte_pktmbuf_append(m_dst, space)) { 3508 CPT_LOG_DP_ERR("Failed to extend " 3509 "mbuf by %uB", space); 3510 ret = -EINVAL; 3511 goto free_mdata_and_exit; 3512 } 3513 3514 params.mac_buf.vaddr = 3515 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3516 params.mac_buf.dma_addr = 3517 rte_pktmbuf_iova_offset(m_dst, off); 3518 params.mac_buf.size = mac_len; 3519 } 3520 } else { 3521 /* Need space for storing generated mac */ 3522 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3523 params.mac_buf.dma_addr = mphys + space; 3524 params.mac_buf.size = mac_len; 3525 space += RTE_ALIGN_CEIL(mac_len, 8); 3526 op[2] = (uintptr_t)params.mac_buf.vaddr; 3527 op[3] = mac_len; 3528 } 3529 3530 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3531 params.meta_buf.dma_addr = mphys + space; 3532 params.meta_buf.size -= space; 3533 3534 /* Out of place processing */ 3535 params.src_iov = (void *)src; 3536 3537 /*Store SG I/O in the api for reuse */ 3538 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3539 CPT_LOG_DP_ERR("Prepare src iov failed"); 3540 ret = -EINVAL; 3541 goto free_mdata_and_exit; 3542 } 3543 3544 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3545 if (unlikely(*prep_req == NULL)) { 3546 ret = -EINVAL; 3547 goto free_mdata_and_exit; 3548 } 3549 3550 *mdata_ptr = mdata; 3551 3552 return 0; 3553 3554 free_mdata_and_exit: 3555 free_op_meta(mdata, m_info->pool); 3556 err_exit: 3557 return ret; 3558 } 3559 3560 #endif /*_CPT_UCODE_H_ */ 3561