1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline void 26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 27 { 28 int i, base; 29 30 for (i = 0; i < 4; i++) { 31 base = 4 * i; 32 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 33 (ck[base + 2] << 8) | (ck[base + 3]); 34 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 35 } 36 } 37 38 static __rte_always_inline int 39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) 40 { 41 uint16_t mac_len = auth->digest_length; 42 int ret; 43 44 switch (auth->algo) { 45 case RTE_CRYPTO_AUTH_MD5: 46 case RTE_CRYPTO_AUTH_MD5_HMAC: 47 ret = (mac_len == 16) ? 0 : -1; 48 break; 49 case RTE_CRYPTO_AUTH_SHA1: 50 case RTE_CRYPTO_AUTH_SHA1_HMAC: 51 ret = (mac_len == 20) ? 0 : -1; 52 break; 53 case RTE_CRYPTO_AUTH_SHA224: 54 case RTE_CRYPTO_AUTH_SHA224_HMAC: 55 ret = (mac_len == 28) ? 0 : -1; 56 break; 57 case RTE_CRYPTO_AUTH_SHA256: 58 case RTE_CRYPTO_AUTH_SHA256_HMAC: 59 ret = (mac_len == 32) ? 0 : -1; 60 break; 61 case RTE_CRYPTO_AUTH_SHA384: 62 case RTE_CRYPTO_AUTH_SHA384_HMAC: 63 ret = (mac_len == 48) ? 0 : -1; 64 break; 65 case RTE_CRYPTO_AUTH_SHA512: 66 case RTE_CRYPTO_AUTH_SHA512_HMAC: 67 ret = (mac_len == 64) ? 0 : -1; 68 break; 69 case RTE_CRYPTO_AUTH_NULL: 70 ret = 0; 71 break; 72 default: 73 ret = -1; 74 } 75 76 return ret; 77 } 78 79 static __rte_always_inline void 80 cpt_fc_salt_update(struct cpt_ctx *cpt_ctx, 81 uint8_t *salt) 82 { 83 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 84 memcpy(fctx->enc.encr_iv, salt, 4); 85 } 86 87 static __rte_always_inline int 88 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 89 { 90 switch (key_len) { 91 case 16: 92 case 24: 93 case 32: 94 return 0; 95 default: 96 return -1; 97 } 98 } 99 100 static __rte_always_inline int 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 102 { 103 int fc_type = 0; 104 switch (type) { 105 case PASSTHROUGH: 106 fc_type = FC_GEN; 107 break; 108 case DES3_CBC: 109 case DES3_ECB: 110 fc_type = FC_GEN; 111 break; 112 case AES_CBC: 113 case AES_ECB: 114 case AES_CFB: 115 case AES_CTR: 116 case AES_GCM: 117 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 118 return -1; 119 fc_type = FC_GEN; 120 break; 121 case CHACHA20: 122 fc_type = FC_GEN; 123 break; 124 case AES_XTS: 125 key_len = key_len / 2; 126 if (unlikely(key_len == 24)) { 127 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 128 return -1; 129 } 130 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 131 return -1; 132 fc_type = FC_GEN; 133 break; 134 case ZUC_EEA3: 135 case SNOW3G_UEA2: 136 if (unlikely(key_len != 16)) 137 return -1; 138 /* No support for AEAD yet */ 139 if (unlikely(ctx->hash_type)) 140 return -1; 141 fc_type = ZUC_SNOW3G; 142 break; 143 case KASUMI_F8_CBC: 144 case KASUMI_F8_ECB: 145 if (unlikely(key_len != 16)) 146 return -1; 147 /* No support for AEAD yet */ 148 if (unlikely(ctx->hash_type)) 149 return -1; 150 fc_type = KASUMI; 151 break; 152 default: 153 return -1; 154 } 155 156 ctx->fc_type = fc_type; 157 return 0; 158 } 159 160 static __rte_always_inline void 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 162 { 163 cpt_ctx->enc_cipher = 0; 164 fctx->enc.enc_cipher = 0; 165 } 166 167 static __rte_always_inline void 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 169 { 170 mc_aes_type_t aes_key_type = 0; 171 switch (key_len) { 172 case 16: 173 aes_key_type = AES_128_BIT; 174 break; 175 case 24: 176 aes_key_type = AES_192_BIT; 177 break; 178 case 32: 179 aes_key_type = AES_256_BIT; 180 break; 181 default: 182 /* This should not happen */ 183 CPT_LOG_DP_ERR("Invalid AES key len"); 184 return; 185 } 186 fctx->enc.aes_key = aes_key_type; 187 } 188 189 static __rte_always_inline void 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 191 uint16_t key_len) 192 { 193 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 194 uint32_t keyx[4]; 195 196 cpt_ctx->snow3g = 1; 197 gen_key_snow3g(key, keyx); 198 memcpy(zs_ctx->ci_key, keyx, key_len); 199 cpt_ctx->zsk_flags = 0; 200 } 201 202 static __rte_always_inline void 203 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 204 uint16_t key_len) 205 { 206 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 207 208 cpt_ctx->snow3g = 0; 209 memcpy(zs_ctx->ci_key, key, key_len); 210 memcpy(zs_ctx->zuc_const, zuc_d, 32); 211 cpt_ctx->zsk_flags = 0; 212 } 213 214 static __rte_always_inline void 215 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 216 uint16_t key_len) 217 { 218 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 219 220 cpt_ctx->k_ecb = 1; 221 memcpy(k_ctx->ci_key, key, key_len); 222 cpt_ctx->zsk_flags = 0; 223 } 224 225 static __rte_always_inline void 226 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 227 uint16_t key_len) 228 { 229 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 230 231 memcpy(k_ctx->ci_key, key, key_len); 232 cpt_ctx->zsk_flags = 0; 233 } 234 235 static __rte_always_inline int 236 cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type, 237 const uint8_t *key, uint16_t key_len, uint8_t *salt) 238 { 239 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 240 int ret; 241 242 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 243 if (unlikely(ret)) 244 return -1; 245 246 if (cpt_ctx->fc_type == FC_GEN) { 247 /* 248 * We need to always say IV is from DPTR as user can 249 * sometimes iverride IV per operation. 250 */ 251 fctx->enc.iv_source = CPT_FROM_DPTR; 252 253 if (cpt_ctx->auth_key_len > 64) 254 return -1; 255 } 256 257 switch (type) { 258 case PASSTHROUGH: 259 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 260 goto success; 261 case DES3_CBC: 262 /* CPT performs DES using 3DES with the 8B DES-key 263 * replicated 2 more times to match the 24B 3DES-key. 264 * Eg. If org. key is "0x0a 0x0b", then new key is 265 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 266 */ 267 if (key_len == 8) { 268 /* Skipping the first 8B as it will be copied 269 * in the regular code flow 270 */ 271 memcpy(fctx->enc.encr_key+key_len, key, key_len); 272 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 273 } 274 break; 275 case DES3_ECB: 276 /* For DES3_ECB IV need to be from CTX. */ 277 fctx->enc.iv_source = CPT_FROM_CTX; 278 break; 279 case AES_CBC: 280 case AES_ECB: 281 case AES_CFB: 282 case AES_CTR: 283 case CHACHA20: 284 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 285 break; 286 case AES_GCM: 287 /* Even though iv source is from dptr, 288 * aes_gcm salt is taken from ctx 289 */ 290 if (salt) { 291 memcpy(fctx->enc.encr_iv, salt, 4); 292 /* Assuming it was just salt update 293 * and nothing else 294 */ 295 if (!key) 296 goto success; 297 } 298 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 299 break; 300 case AES_XTS: 301 key_len = key_len / 2; 302 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 303 304 /* Copy key2 for XTS into ipad */ 305 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 306 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 307 break; 308 case SNOW3G_UEA2: 309 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 310 goto success; 311 case ZUC_EEA3: 312 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 313 goto success; 314 case KASUMI_F8_ECB: 315 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 316 goto success; 317 case KASUMI_F8_CBC: 318 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 319 goto success; 320 default: 321 return -1; 322 } 323 324 /* Only for FC_GEN case */ 325 326 /* For GMAC auth, cipher must be NULL */ 327 if (cpt_ctx->hash_type != GMAC_TYPE) 328 fctx->enc.enc_cipher = type; 329 330 memcpy(fctx->enc.encr_key, key, key_len); 331 332 success: 333 cpt_ctx->enc_cipher = type; 334 335 return 0; 336 } 337 338 static __rte_always_inline uint32_t 339 fill_sg_comp(sg_comp_t *list, 340 uint32_t i, 341 phys_addr_t dma_addr, 342 uint32_t size) 343 { 344 sg_comp_t *to = &list[i>>2]; 345 346 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 347 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 348 i++; 349 return i; 350 } 351 352 static __rte_always_inline uint32_t 353 fill_sg_comp_from_buf(sg_comp_t *list, 354 uint32_t i, 355 buf_ptr_t *from) 356 { 357 sg_comp_t *to = &list[i>>2]; 358 359 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 360 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 361 i++; 362 return i; 363 } 364 365 static __rte_always_inline uint32_t 366 fill_sg_comp_from_buf_min(sg_comp_t *list, 367 uint32_t i, 368 buf_ptr_t *from, 369 uint32_t *psize) 370 { 371 sg_comp_t *to = &list[i >> 2]; 372 uint32_t size = *psize; 373 uint32_t e_len; 374 375 e_len = (size > from->size) ? from->size : size; 376 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 377 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 378 *psize -= e_len; 379 i++; 380 return i; 381 } 382 383 /* 384 * This fills the MC expected SGIO list 385 * from IOV given by user. 386 */ 387 static __rte_always_inline uint32_t 388 fill_sg_comp_from_iov(sg_comp_t *list, 389 uint32_t i, 390 iov_ptr_t *from, uint32_t from_offset, 391 uint32_t *psize, buf_ptr_t *extra_buf, 392 uint32_t extra_offset) 393 { 394 int32_t j; 395 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 396 uint32_t size = *psize; 397 buf_ptr_t *bufs; 398 399 bufs = from->bufs; 400 for (j = 0; (j < from->buf_cnt) && size; j++) { 401 phys_addr_t e_dma_addr; 402 uint32_t e_len; 403 sg_comp_t *to = &list[i >> 2]; 404 405 if (unlikely(from_offset)) { 406 if (from_offset >= bufs[j].size) { 407 from_offset -= bufs[j].size; 408 continue; 409 } 410 e_dma_addr = bufs[j].dma_addr + from_offset; 411 e_len = (size > (bufs[j].size - from_offset)) ? 412 (bufs[j].size - from_offset) : size; 413 from_offset = 0; 414 } else { 415 e_dma_addr = bufs[j].dma_addr; 416 e_len = (size > bufs[j].size) ? 417 bufs[j].size : size; 418 } 419 420 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 421 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 422 423 if (extra_len && (e_len >= extra_offset)) { 424 /* Break the data at given offset */ 425 uint32_t next_len = e_len - extra_offset; 426 phys_addr_t next_dma = e_dma_addr + extra_offset; 427 428 if (!extra_offset) { 429 i--; 430 } else { 431 e_len = extra_offset; 432 size -= e_len; 433 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 434 } 435 436 extra_len = RTE_MIN(extra_len, size); 437 /* Insert extra data ptr */ 438 if (extra_len) { 439 i++; 440 to = &list[i >> 2]; 441 to->u.s.len[i % 4] = 442 rte_cpu_to_be_16(extra_len); 443 to->ptr[i % 4] = 444 rte_cpu_to_be_64(extra_buf->dma_addr); 445 size -= extra_len; 446 } 447 448 next_len = RTE_MIN(next_len, size); 449 /* insert the rest of the data */ 450 if (next_len) { 451 i++; 452 to = &list[i >> 2]; 453 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 454 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 455 size -= next_len; 456 } 457 extra_len = 0; 458 459 } else { 460 size -= e_len; 461 } 462 if (extra_offset) 463 extra_offset -= size; 464 i++; 465 } 466 467 *psize = size; 468 return (uint32_t)i; 469 } 470 471 static __rte_always_inline void 472 cpt_digest_gen_prep(uint32_t flags, 473 uint64_t d_lens, 474 digest_params_t *params, 475 void *op, 476 void **prep_req) 477 { 478 struct cpt_request_info *req; 479 uint32_t size, i; 480 uint16_t data_len, mac_len, key_len; 481 auth_type_t hash_type; 482 buf_ptr_t *meta_p; 483 struct cpt_ctx *ctx; 484 sg_comp_t *gather_comp; 485 sg_comp_t *scatter_comp; 486 uint8_t *in_buffer; 487 uint32_t g_size_bytes, s_size_bytes; 488 uint64_t dptr_dma, rptr_dma; 489 vq_cmd_word0_t vq_cmd_w0; 490 void *c_vaddr, *m_vaddr; 491 uint64_t c_dma, m_dma; 492 493 ctx = params->ctx_buf.vaddr; 494 meta_p = ¶ms->meta_buf; 495 496 m_vaddr = meta_p->vaddr; 497 m_dma = meta_p->dma_addr; 498 499 /* 500 * Save initial space that followed app data for completion code & 501 * alternate completion code to fall in same cache line as app data 502 */ 503 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 504 m_dma += COMPLETION_CODE_SIZE; 505 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 506 (uint8_t *)m_vaddr; 507 c_vaddr = (uint8_t *)m_vaddr + size; 508 c_dma = m_dma + size; 509 size += sizeof(cpt_res_s_t); 510 511 m_vaddr = (uint8_t *)m_vaddr + size; 512 m_dma += size; 513 514 req = m_vaddr; 515 516 size = sizeof(struct cpt_request_info); 517 m_vaddr = (uint8_t *)m_vaddr + size; 518 m_dma += size; 519 520 hash_type = ctx->hash_type; 521 mac_len = ctx->mac_len; 522 key_len = ctx->auth_key_len; 523 data_len = AUTH_DLEN(d_lens); 524 525 /*GP op header */ 526 vq_cmd_w0.s.opcode.minor = 0; 527 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 528 if (ctx->hmac) { 529 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 530 vq_cmd_w0.s.param1 = key_len; 531 vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8); 532 } else { 533 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 534 vq_cmd_w0.s.param1 = 0; 535 vq_cmd_w0.s.dlen = data_len; 536 } 537 538 /* Null auth only case enters the if */ 539 if (unlikely(!hash_type && !ctx->enc_cipher)) { 540 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC; 541 /* Minor op is passthrough */ 542 vq_cmd_w0.s.opcode.minor = 0x03; 543 /* Send out completion code only */ 544 vq_cmd_w0.s.param2 = 0x1; 545 } 546 547 /* DPTR has SG list */ 548 in_buffer = m_vaddr; 549 dptr_dma = m_dma; 550 551 ((uint16_t *)in_buffer)[0] = 0; 552 ((uint16_t *)in_buffer)[1] = 0; 553 554 /* TODO Add error check if space will be sufficient */ 555 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 556 557 /* 558 * Input gather list 559 */ 560 561 i = 0; 562 563 if (ctx->hmac) { 564 uint64_t k_dma = params->ctx_buf.dma_addr + 565 offsetof(struct cpt_ctx, auth_key); 566 /* Key */ 567 i = fill_sg_comp(gather_comp, i, k_dma, 568 RTE_ALIGN_CEIL(key_len, 8)); 569 } 570 571 /* input data */ 572 size = data_len; 573 if (size) { 574 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 575 0, &size, NULL, 0); 576 if (unlikely(size)) { 577 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 578 " by %dB", size); 579 return; 580 } 581 } else { 582 /* 583 * Looks like we need to support zero data 584 * gather ptr in case of hash & hmac 585 */ 586 i++; 587 } 588 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 589 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 590 591 /* 592 * Output Gather list 593 */ 594 595 i = 0; 596 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 597 598 if (flags & VALID_MAC_BUF) { 599 if (unlikely(params->mac_buf.size < mac_len)) { 600 CPT_LOG_DP_ERR("Insufficient MAC size"); 601 return; 602 } 603 604 size = mac_len; 605 i = fill_sg_comp_from_buf_min(scatter_comp, i, 606 ¶ms->mac_buf, &size); 607 } else { 608 size = mac_len; 609 i = fill_sg_comp_from_iov(scatter_comp, i, 610 params->src_iov, data_len, 611 &size, NULL, 0); 612 if (unlikely(size)) { 613 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 614 " %dB", size); 615 return; 616 } 617 } 618 619 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 620 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 621 622 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 623 624 /* This is DPTR len incase of SG mode */ 625 vq_cmd_w0.s.dlen = size; 626 627 m_vaddr = (uint8_t *)m_vaddr + size; 628 m_dma += size; 629 630 /* cpt alternate completion address saved earlier */ 631 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 632 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 633 rptr_dma = c_dma - 8; 634 635 req->ist.ei1 = dptr_dma; 636 req->ist.ei2 = rptr_dma; 637 638 /* 16 byte aligned cpt res address */ 639 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 640 *req->completion_addr = COMPLETION_CODE_INIT; 641 req->comp_baddr = c_dma; 642 643 /* Fill microcode part of instruction */ 644 req->ist.ei0 = vq_cmd_w0.u64; 645 646 req->op = op; 647 648 *prep_req = req; 649 return; 650 } 651 652 static __rte_always_inline void 653 cpt_enc_hmac_prep(uint32_t flags, 654 uint64_t d_offs, 655 uint64_t d_lens, 656 fc_params_t *fc_params, 657 void *op, 658 void **prep_req) 659 { 660 uint32_t iv_offset = 0; 661 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 662 struct cpt_ctx *cpt_ctx; 663 uint32_t cipher_type, hash_type; 664 uint32_t mac_len, size; 665 uint8_t iv_len = 16; 666 struct cpt_request_info *req; 667 buf_ptr_t *meta_p, *aad_buf = NULL; 668 uint32_t encr_offset, auth_offset; 669 uint32_t encr_data_len, auth_data_len, aad_len = 0; 670 uint32_t passthrough_len = 0; 671 void *m_vaddr, *offset_vaddr; 672 uint64_t m_dma, offset_dma; 673 vq_cmd_word0_t vq_cmd_w0; 674 void *c_vaddr; 675 uint64_t c_dma; 676 677 meta_p = &fc_params->meta_buf; 678 m_vaddr = meta_p->vaddr; 679 m_dma = meta_p->dma_addr; 680 681 encr_offset = ENCR_OFFSET(d_offs); 682 auth_offset = AUTH_OFFSET(d_offs); 683 encr_data_len = ENCR_DLEN(d_lens); 684 auth_data_len = AUTH_DLEN(d_lens); 685 if (unlikely(flags & VALID_AAD_BUF)) { 686 /* 687 * We dont support both aad 688 * and auth data separately 689 */ 690 auth_data_len = 0; 691 auth_offset = 0; 692 aad_len = fc_params->aad_buf.size; 693 aad_buf = &fc_params->aad_buf; 694 } 695 cpt_ctx = fc_params->ctx_buf.vaddr; 696 cipher_type = cpt_ctx->enc_cipher; 697 hash_type = cpt_ctx->hash_type; 698 mac_len = cpt_ctx->mac_len; 699 700 /* 701 * Save initial space that followed app data for completion code & 702 * alternate completion code to fall in same cache line as app data 703 */ 704 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 705 m_dma += COMPLETION_CODE_SIZE; 706 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 707 (uint8_t *)m_vaddr; 708 709 c_vaddr = (uint8_t *)m_vaddr + size; 710 c_dma = m_dma + size; 711 size += sizeof(cpt_res_s_t); 712 713 m_vaddr = (uint8_t *)m_vaddr + size; 714 m_dma += size; 715 716 /* start cpt request info struct at 8 byte boundary */ 717 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 718 (uint8_t *)m_vaddr; 719 720 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 721 722 size += sizeof(struct cpt_request_info); 723 m_vaddr = (uint8_t *)m_vaddr + size; 724 m_dma += size; 725 726 if (unlikely(!(flags & VALID_IV_BUF))) { 727 iv_len = 0; 728 iv_offset = ENCR_IV_OFFSET(d_offs); 729 } 730 731 if (unlikely(flags & VALID_AAD_BUF)) { 732 /* 733 * When AAD is given, data above encr_offset is pass through 734 * Since AAD is given as separate pointer and not as offset, 735 * this is a special case as we need to fragment input data 736 * into passthrough + encr_data and then insert AAD in between. 737 */ 738 if (hash_type != GMAC_TYPE) { 739 passthrough_len = encr_offset; 740 auth_offset = passthrough_len + iv_len; 741 encr_offset = passthrough_len + aad_len + iv_len; 742 auth_data_len = aad_len + encr_data_len; 743 } else { 744 passthrough_len = 16 + aad_len; 745 auth_offset = passthrough_len + iv_len; 746 auth_data_len = aad_len; 747 } 748 } else { 749 encr_offset += iv_len; 750 auth_offset += iv_len; 751 } 752 753 /* Encryption */ 754 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 755 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT; 756 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc << 757 CPT_HMAC_FIRST_BIT_POS); 758 759 if (hash_type == GMAC_TYPE) { 760 encr_offset = 0; 761 encr_data_len = 0; 762 } 763 764 auth_dlen = auth_offset + auth_data_len; 765 enc_dlen = encr_data_len + encr_offset; 766 if (unlikely(encr_data_len & 0xf)) { 767 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 768 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 769 encr_offset; 770 else if (likely((cipher_type == AES_CBC) || 771 (cipher_type == AES_ECB))) 772 enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + 773 encr_offset; 774 } 775 776 if (unlikely(auth_dlen > enc_dlen)) { 777 inputlen = auth_dlen; 778 outputlen = auth_dlen + mac_len; 779 } else { 780 inputlen = enc_dlen; 781 outputlen = enc_dlen + mac_len; 782 } 783 784 if (cpt_ctx->auth_enc != 0) 785 outputlen = enc_dlen; 786 787 /* GP op header */ 788 vq_cmd_w0.s.param1 = encr_data_len; 789 vq_cmd_w0.s.param2 = auth_data_len; 790 /* 791 * In 83XX since we have a limitation of 792 * IV & Offset control word not part of instruction 793 * and need to be part of Data Buffer, we check if 794 * head room is there and then only do the Direct mode processing 795 */ 796 if (likely((flags & SINGLE_BUF_INPLACE) && 797 (flags & SINGLE_BUF_HEADTAILROOM))) { 798 void *dm_vaddr = fc_params->bufs[0].vaddr; 799 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 800 /* 801 * This flag indicates that there is 24 bytes head room and 802 * 8 bytes tail room available, so that we get to do 803 * DIRECT MODE with limitation 804 */ 805 806 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 807 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 808 809 /* DPTR */ 810 req->ist.ei1 = offset_dma; 811 /* RPTR should just exclude offset control word */ 812 req->ist.ei2 = dm_dma_addr - iv_len; 813 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 814 + outputlen - iv_len); 815 816 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 817 818 if (likely(iv_len)) { 819 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 820 + OFF_CTRL_LEN); 821 uint64_t *src = fc_params->iv_buf; 822 dest[0] = src[0]; 823 dest[1] = src[1]; 824 } 825 826 *(uint64_t *)offset_vaddr = 827 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 828 ((uint64_t)iv_offset << 8) | 829 ((uint64_t)auth_offset)); 830 831 } else { 832 uint32_t i, g_size_bytes, s_size_bytes; 833 uint64_t dptr_dma, rptr_dma; 834 sg_comp_t *gather_comp; 835 sg_comp_t *scatter_comp; 836 uint8_t *in_buffer; 837 838 /* This falls under strict SG mode */ 839 offset_vaddr = m_vaddr; 840 offset_dma = m_dma; 841 size = OFF_CTRL_LEN + iv_len; 842 843 m_vaddr = (uint8_t *)m_vaddr + size; 844 m_dma += size; 845 846 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 847 848 if (likely(iv_len)) { 849 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 850 + OFF_CTRL_LEN); 851 uint64_t *src = fc_params->iv_buf; 852 dest[0] = src[0]; 853 dest[1] = src[1]; 854 } 855 856 *(uint64_t *)offset_vaddr = 857 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 858 ((uint64_t)iv_offset << 8) | 859 ((uint64_t)auth_offset)); 860 861 /* DPTR has SG list */ 862 in_buffer = m_vaddr; 863 dptr_dma = m_dma; 864 865 ((uint16_t *)in_buffer)[0] = 0; 866 ((uint16_t *)in_buffer)[1] = 0; 867 868 /* TODO Add error check if space will be sufficient */ 869 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 870 871 /* 872 * Input Gather List 873 */ 874 875 i = 0; 876 877 /* Offset control word that includes iv */ 878 i = fill_sg_comp(gather_comp, i, offset_dma, 879 OFF_CTRL_LEN + iv_len); 880 881 /* Add input data */ 882 size = inputlen - iv_len; 883 if (likely(size)) { 884 uint32_t aad_offset = aad_len ? passthrough_len : 0; 885 886 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 887 i = fill_sg_comp_from_buf_min(gather_comp, i, 888 fc_params->bufs, 889 &size); 890 } else { 891 i = fill_sg_comp_from_iov(gather_comp, i, 892 fc_params->src_iov, 893 0, &size, 894 aad_buf, aad_offset); 895 } 896 897 if (unlikely(size)) { 898 CPT_LOG_DP_ERR("Insufficient buffer space," 899 " size %d needed", size); 900 return; 901 } 902 } 903 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 904 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 905 906 /* 907 * Output Scatter list 908 */ 909 i = 0; 910 scatter_comp = 911 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 912 913 /* Add IV */ 914 if (likely(iv_len)) { 915 i = fill_sg_comp(scatter_comp, i, 916 offset_dma + OFF_CTRL_LEN, 917 iv_len); 918 } 919 920 /* output data or output data + digest*/ 921 if (unlikely(flags & VALID_MAC_BUF)) { 922 size = outputlen - iv_len - mac_len; 923 if (size) { 924 uint32_t aad_offset = 925 aad_len ? passthrough_len : 0; 926 927 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 928 i = fill_sg_comp_from_buf_min( 929 scatter_comp, 930 i, 931 fc_params->bufs, 932 &size); 933 } else { 934 i = fill_sg_comp_from_iov(scatter_comp, 935 i, 936 fc_params->dst_iov, 937 0, 938 &size, 939 aad_buf, 940 aad_offset); 941 } 942 if (unlikely(size)) { 943 CPT_LOG_DP_ERR("Insufficient buffer" 944 " space, size %d needed", 945 size); 946 return; 947 } 948 } 949 /* mac_data */ 950 if (mac_len) { 951 i = fill_sg_comp_from_buf(scatter_comp, i, 952 &fc_params->mac_buf); 953 } 954 } else { 955 /* Output including mac */ 956 size = outputlen - iv_len; 957 if (likely(size)) { 958 uint32_t aad_offset = 959 aad_len ? passthrough_len : 0; 960 961 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 962 i = fill_sg_comp_from_buf_min( 963 scatter_comp, 964 i, 965 fc_params->bufs, 966 &size); 967 } else { 968 i = fill_sg_comp_from_iov(scatter_comp, 969 i, 970 fc_params->dst_iov, 971 0, 972 &size, 973 aad_buf, 974 aad_offset); 975 } 976 if (unlikely(size)) { 977 CPT_LOG_DP_ERR("Insufficient buffer" 978 " space, size %d needed", 979 size); 980 return; 981 } 982 } 983 } 984 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 985 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 986 987 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 988 989 /* This is DPTR len incase of SG mode */ 990 vq_cmd_w0.s.dlen = size; 991 992 m_vaddr = (uint8_t *)m_vaddr + size; 993 m_dma += size; 994 995 /* cpt alternate completion address saved earlier */ 996 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 997 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 998 rptr_dma = c_dma - 8; 999 1000 req->ist.ei1 = dptr_dma; 1001 req->ist.ei2 = rptr_dma; 1002 } 1003 1004 /* 16 byte aligned cpt res address */ 1005 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1006 *req->completion_addr = COMPLETION_CODE_INIT; 1007 req->comp_baddr = c_dma; 1008 1009 /* Fill microcode part of instruction */ 1010 req->ist.ei0 = vq_cmd_w0.u64; 1011 1012 req->op = op; 1013 1014 *prep_req = req; 1015 return; 1016 } 1017 1018 static __rte_always_inline void 1019 cpt_dec_hmac_prep(uint32_t flags, 1020 uint64_t d_offs, 1021 uint64_t d_lens, 1022 fc_params_t *fc_params, 1023 void *op, 1024 void **prep_req) 1025 { 1026 uint32_t iv_offset = 0, size; 1027 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1028 struct cpt_ctx *cpt_ctx; 1029 int32_t hash_type, mac_len; 1030 uint8_t iv_len = 16; 1031 struct cpt_request_info *req; 1032 buf_ptr_t *meta_p, *aad_buf = NULL; 1033 uint32_t encr_offset, auth_offset; 1034 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1035 uint32_t passthrough_len = 0; 1036 void *m_vaddr, *offset_vaddr; 1037 uint64_t m_dma, offset_dma; 1038 vq_cmd_word0_t vq_cmd_w0; 1039 void *c_vaddr; 1040 uint64_t c_dma; 1041 1042 meta_p = &fc_params->meta_buf; 1043 m_vaddr = meta_p->vaddr; 1044 m_dma = meta_p->dma_addr; 1045 1046 encr_offset = ENCR_OFFSET(d_offs); 1047 auth_offset = AUTH_OFFSET(d_offs); 1048 encr_data_len = ENCR_DLEN(d_lens); 1049 auth_data_len = AUTH_DLEN(d_lens); 1050 1051 if (unlikely(flags & VALID_AAD_BUF)) { 1052 /* 1053 * We dont support both aad 1054 * and auth data separately 1055 */ 1056 auth_data_len = 0; 1057 auth_offset = 0; 1058 aad_len = fc_params->aad_buf.size; 1059 aad_buf = &fc_params->aad_buf; 1060 } 1061 1062 cpt_ctx = fc_params->ctx_buf.vaddr; 1063 hash_type = cpt_ctx->hash_type; 1064 mac_len = cpt_ctx->mac_len; 1065 1066 if (unlikely(!(flags & VALID_IV_BUF))) { 1067 iv_len = 0; 1068 iv_offset = ENCR_IV_OFFSET(d_offs); 1069 } 1070 1071 if (unlikely(flags & VALID_AAD_BUF)) { 1072 /* 1073 * When AAD is given, data above encr_offset is pass through 1074 * Since AAD is given as separate pointer and not as offset, 1075 * this is a special case as we need to fragment input data 1076 * into passthrough + encr_data and then insert AAD in between. 1077 */ 1078 if (hash_type != GMAC_TYPE) { 1079 passthrough_len = encr_offset; 1080 auth_offset = passthrough_len + iv_len; 1081 encr_offset = passthrough_len + aad_len + iv_len; 1082 auth_data_len = aad_len + encr_data_len; 1083 } else { 1084 passthrough_len = 16 + aad_len; 1085 auth_offset = passthrough_len + iv_len; 1086 auth_data_len = aad_len; 1087 } 1088 } else { 1089 encr_offset += iv_len; 1090 auth_offset += iv_len; 1091 } 1092 1093 /* 1094 * Save initial space that followed app data for completion code & 1095 * alternate completion code to fall in same cache line as app data 1096 */ 1097 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1098 m_dma += COMPLETION_CODE_SIZE; 1099 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1100 (uint8_t *)m_vaddr; 1101 c_vaddr = (uint8_t *)m_vaddr + size; 1102 c_dma = m_dma + size; 1103 size += sizeof(cpt_res_s_t); 1104 1105 m_vaddr = (uint8_t *)m_vaddr + size; 1106 m_dma += size; 1107 1108 /* start cpt request info structure at 8 byte alignment */ 1109 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1110 (uint8_t *)m_vaddr; 1111 1112 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1113 1114 size += sizeof(struct cpt_request_info); 1115 m_vaddr = (uint8_t *)m_vaddr + size; 1116 m_dma += size; 1117 1118 /* Decryption */ 1119 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; 1120 vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT; 1121 vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth << 1122 CPT_HMAC_FIRST_BIT_POS); 1123 1124 if (hash_type == GMAC_TYPE) { 1125 encr_offset = 0; 1126 encr_data_len = 0; 1127 } 1128 1129 enc_dlen = encr_offset + encr_data_len; 1130 auth_dlen = auth_offset + auth_data_len; 1131 1132 if (auth_dlen > enc_dlen) { 1133 inputlen = auth_dlen + mac_len; 1134 outputlen = auth_dlen; 1135 } else { 1136 inputlen = enc_dlen + mac_len; 1137 outputlen = enc_dlen; 1138 } 1139 1140 if (cpt_ctx->dec_auth != 0) 1141 outputlen = inputlen = enc_dlen; 1142 1143 vq_cmd_w0.s.param1 = encr_data_len; 1144 vq_cmd_w0.s.param2 = auth_data_len; 1145 1146 /* 1147 * In 83XX since we have a limitation of 1148 * IV & Offset control word not part of instruction 1149 * and need to be part of Data Buffer, we check if 1150 * head room is there and then only do the Direct mode processing 1151 */ 1152 if (likely((flags & SINGLE_BUF_INPLACE) && 1153 (flags & SINGLE_BUF_HEADTAILROOM))) { 1154 void *dm_vaddr = fc_params->bufs[0].vaddr; 1155 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1156 /* 1157 * This flag indicates that there is 24 bytes head room and 1158 * 8 bytes tail room available, so that we get to do 1159 * DIRECT MODE with limitation 1160 */ 1161 1162 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1163 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1164 req->ist.ei1 = offset_dma; 1165 1166 /* RPTR should just exclude offset control word */ 1167 req->ist.ei2 = dm_dma_addr - iv_len; 1168 1169 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1170 outputlen - iv_len); 1171 /* since this is decryption, 1172 * don't touch the content of 1173 * alternate ccode space as it contains 1174 * hmac. 1175 */ 1176 1177 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1178 1179 if (likely(iv_len)) { 1180 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1181 OFF_CTRL_LEN); 1182 uint64_t *src = fc_params->iv_buf; 1183 dest[0] = src[0]; 1184 dest[1] = src[1]; 1185 } 1186 1187 *(uint64_t *)offset_vaddr = 1188 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1189 ((uint64_t)iv_offset << 8) | 1190 ((uint64_t)auth_offset)); 1191 1192 } else { 1193 uint64_t dptr_dma, rptr_dma; 1194 uint32_t g_size_bytes, s_size_bytes; 1195 sg_comp_t *gather_comp; 1196 sg_comp_t *scatter_comp; 1197 uint8_t *in_buffer; 1198 uint8_t i = 0; 1199 1200 /* This falls under strict SG mode */ 1201 offset_vaddr = m_vaddr; 1202 offset_dma = m_dma; 1203 size = OFF_CTRL_LEN + iv_len; 1204 1205 m_vaddr = (uint8_t *)m_vaddr + size; 1206 m_dma += size; 1207 1208 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1209 1210 if (likely(iv_len)) { 1211 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1212 OFF_CTRL_LEN); 1213 uint64_t *src = fc_params->iv_buf; 1214 dest[0] = src[0]; 1215 dest[1] = src[1]; 1216 } 1217 1218 *(uint64_t *)offset_vaddr = 1219 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1220 ((uint64_t)iv_offset << 8) | 1221 ((uint64_t)auth_offset)); 1222 1223 /* DPTR has SG list */ 1224 in_buffer = m_vaddr; 1225 dptr_dma = m_dma; 1226 1227 ((uint16_t *)in_buffer)[0] = 0; 1228 ((uint16_t *)in_buffer)[1] = 0; 1229 1230 /* TODO Add error check if space will be sufficient */ 1231 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1232 1233 /* 1234 * Input Gather List 1235 */ 1236 i = 0; 1237 1238 /* Offset control word that includes iv */ 1239 i = fill_sg_comp(gather_comp, i, offset_dma, 1240 OFF_CTRL_LEN + iv_len); 1241 1242 /* Add input data */ 1243 if (flags & VALID_MAC_BUF) { 1244 size = inputlen - iv_len - mac_len; 1245 if (size) { 1246 /* input data only */ 1247 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1248 i = fill_sg_comp_from_buf_min( 1249 gather_comp, i, 1250 fc_params->bufs, 1251 &size); 1252 } else { 1253 uint32_t aad_offset = aad_len ? 1254 passthrough_len : 0; 1255 1256 i = fill_sg_comp_from_iov(gather_comp, 1257 i, 1258 fc_params->src_iov, 1259 0, &size, 1260 aad_buf, 1261 aad_offset); 1262 } 1263 if (unlikely(size)) { 1264 CPT_LOG_DP_ERR("Insufficient buffer" 1265 " space, size %d needed", 1266 size); 1267 return; 1268 } 1269 } 1270 1271 /* mac data */ 1272 if (mac_len) { 1273 i = fill_sg_comp_from_buf(gather_comp, i, 1274 &fc_params->mac_buf); 1275 } 1276 } else { 1277 /* input data + mac */ 1278 size = inputlen - iv_len; 1279 if (size) { 1280 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1281 i = fill_sg_comp_from_buf_min( 1282 gather_comp, i, 1283 fc_params->bufs, 1284 &size); 1285 } else { 1286 uint32_t aad_offset = aad_len ? 1287 passthrough_len : 0; 1288 1289 if (unlikely(!fc_params->src_iov)) { 1290 CPT_LOG_DP_ERR("Bad input args"); 1291 return; 1292 } 1293 1294 i = fill_sg_comp_from_iov( 1295 gather_comp, i, 1296 fc_params->src_iov, 1297 0, &size, 1298 aad_buf, 1299 aad_offset); 1300 } 1301 1302 if (unlikely(size)) { 1303 CPT_LOG_DP_ERR("Insufficient buffer" 1304 " space, size %d needed", 1305 size); 1306 return; 1307 } 1308 } 1309 } 1310 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1311 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1312 1313 /* 1314 * Output Scatter List 1315 */ 1316 1317 i = 0; 1318 scatter_comp = 1319 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1320 1321 /* Add iv */ 1322 if (iv_len) { 1323 i = fill_sg_comp(scatter_comp, i, 1324 offset_dma + OFF_CTRL_LEN, 1325 iv_len); 1326 } 1327 1328 /* Add output data */ 1329 size = outputlen - iv_len; 1330 if (size) { 1331 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1332 /* handle single buffer here */ 1333 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1334 fc_params->bufs, 1335 &size); 1336 } else { 1337 uint32_t aad_offset = aad_len ? 1338 passthrough_len : 0; 1339 1340 if (unlikely(!fc_params->dst_iov)) { 1341 CPT_LOG_DP_ERR("Bad input args"); 1342 return; 1343 } 1344 1345 i = fill_sg_comp_from_iov(scatter_comp, i, 1346 fc_params->dst_iov, 0, 1347 &size, aad_buf, 1348 aad_offset); 1349 } 1350 1351 if (unlikely(size)) { 1352 CPT_LOG_DP_ERR("Insufficient buffer space," 1353 " size %d needed", size); 1354 return; 1355 } 1356 } 1357 1358 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1359 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1360 1361 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1362 1363 /* This is DPTR len incase of SG mode */ 1364 vq_cmd_w0.s.dlen = size; 1365 1366 m_vaddr = (uint8_t *)m_vaddr + size; 1367 m_dma += size; 1368 1369 /* cpt alternate completion address saved earlier */ 1370 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1371 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1372 rptr_dma = c_dma - 8; 1373 size += COMPLETION_CODE_SIZE; 1374 1375 req->ist.ei1 = dptr_dma; 1376 req->ist.ei2 = rptr_dma; 1377 } 1378 1379 /* 16 byte aligned cpt res address */ 1380 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1381 *req->completion_addr = COMPLETION_CODE_INIT; 1382 req->comp_baddr = c_dma; 1383 1384 /* Fill microcode part of instruction */ 1385 req->ist.ei0 = vq_cmd_w0.u64; 1386 1387 req->op = op; 1388 1389 *prep_req = req; 1390 return; 1391 } 1392 1393 static __rte_always_inline void 1394 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1395 uint64_t d_offs, 1396 uint64_t d_lens, 1397 fc_params_t *params, 1398 void *op, 1399 void **prep_req) 1400 { 1401 uint32_t size; 1402 int32_t inputlen, outputlen; 1403 struct cpt_ctx *cpt_ctx; 1404 uint32_t mac_len = 0; 1405 uint8_t snow3g, j; 1406 struct cpt_request_info *req; 1407 buf_ptr_t *buf_p; 1408 uint32_t encr_offset = 0, auth_offset = 0; 1409 uint32_t encr_data_len = 0, auth_data_len = 0; 1410 int flags, iv_len = 16; 1411 void *m_vaddr, *c_vaddr; 1412 uint64_t m_dma, c_dma, offset_ctrl; 1413 uint64_t *offset_vaddr, offset_dma; 1414 uint32_t *iv_s, iv[4]; 1415 vq_cmd_word0_t vq_cmd_w0; 1416 1417 buf_p = ¶ms->meta_buf; 1418 m_vaddr = buf_p->vaddr; 1419 m_dma = buf_p->dma_addr; 1420 1421 cpt_ctx = params->ctx_buf.vaddr; 1422 flags = cpt_ctx->zsk_flags; 1423 mac_len = cpt_ctx->mac_len; 1424 snow3g = cpt_ctx->snow3g; 1425 1426 /* 1427 * Save initial space that followed app data for completion code & 1428 * alternate completion code to fall in same cache line as app data 1429 */ 1430 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1431 m_dma += COMPLETION_CODE_SIZE; 1432 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1433 (uint8_t *)m_vaddr; 1434 1435 c_vaddr = (uint8_t *)m_vaddr + size; 1436 c_dma = m_dma + size; 1437 size += sizeof(cpt_res_s_t); 1438 1439 m_vaddr = (uint8_t *)m_vaddr + size; 1440 m_dma += size; 1441 1442 /* Reserve memory for cpt request info */ 1443 req = m_vaddr; 1444 1445 size = sizeof(struct cpt_request_info); 1446 m_vaddr = (uint8_t *)m_vaddr + size; 1447 m_dma += size; 1448 1449 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1450 1451 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1452 1453 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1454 (0 << 3) | (flags & 0x7)); 1455 1456 if (flags == 0x1) { 1457 /* 1458 * Microcode expects offsets in bytes 1459 * TODO: Rounding off 1460 */ 1461 auth_data_len = AUTH_DLEN(d_lens); 1462 1463 /* EIA3 or UIA2 */ 1464 auth_offset = AUTH_OFFSET(d_offs); 1465 auth_offset = auth_offset / 8; 1466 1467 /* consider iv len */ 1468 auth_offset += iv_len; 1469 1470 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1471 outputlen = mac_len; 1472 1473 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1474 1475 } else { 1476 /* EEA3 or UEA2 */ 1477 /* 1478 * Microcode expects offsets in bytes 1479 * TODO: Rounding off 1480 */ 1481 encr_data_len = ENCR_DLEN(d_lens); 1482 1483 encr_offset = ENCR_OFFSET(d_offs); 1484 encr_offset = encr_offset / 8; 1485 /* consider iv len */ 1486 encr_offset += iv_len; 1487 1488 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1489 outputlen = inputlen; 1490 1491 /* iv offset is 0 */ 1492 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1493 } 1494 1495 /* IV */ 1496 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1497 params->iv_buf; 1498 1499 if (snow3g) { 1500 /* 1501 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1502 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1503 */ 1504 1505 for (j = 0; j < 4; j++) 1506 iv[j] = iv_s[3 - j]; 1507 } else { 1508 /* ZUC doesn't need a swap */ 1509 for (j = 0; j < 4; j++) 1510 iv[j] = iv_s[j]; 1511 } 1512 1513 /* 1514 * GP op header, lengths are expected in bits. 1515 */ 1516 vq_cmd_w0.s.param1 = encr_data_len; 1517 vq_cmd_w0.s.param2 = auth_data_len; 1518 1519 /* 1520 * In 83XX since we have a limitation of 1521 * IV & Offset control word not part of instruction 1522 * and need to be part of Data Buffer, we check if 1523 * head room is there and then only do the Direct mode processing 1524 */ 1525 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1526 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1527 void *dm_vaddr = params->bufs[0].vaddr; 1528 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1529 /* 1530 * This flag indicates that there is 24 bytes head room and 1531 * 8 bytes tail room available, so that we get to do 1532 * DIRECT MODE with limitation 1533 */ 1534 1535 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1536 OFF_CTRL_LEN - iv_len); 1537 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1538 1539 /* DPTR */ 1540 req->ist.ei1 = offset_dma; 1541 /* RPTR should just exclude offset control word */ 1542 req->ist.ei2 = dm_dma_addr - iv_len; 1543 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1544 + outputlen - iv_len); 1545 1546 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1547 1548 if (likely(iv_len)) { 1549 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1550 + OFF_CTRL_LEN); 1551 memcpy(iv_d, iv, 16); 1552 } 1553 1554 *offset_vaddr = offset_ctrl; 1555 } else { 1556 uint32_t i, g_size_bytes, s_size_bytes; 1557 uint64_t dptr_dma, rptr_dma; 1558 sg_comp_t *gather_comp; 1559 sg_comp_t *scatter_comp; 1560 uint8_t *in_buffer; 1561 uint32_t *iv_d; 1562 1563 /* save space for iv */ 1564 offset_vaddr = m_vaddr; 1565 offset_dma = m_dma; 1566 1567 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1568 m_dma += OFF_CTRL_LEN + iv_len; 1569 1570 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1571 1572 /* DPTR has SG list */ 1573 in_buffer = m_vaddr; 1574 dptr_dma = m_dma; 1575 1576 ((uint16_t *)in_buffer)[0] = 0; 1577 ((uint16_t *)in_buffer)[1] = 0; 1578 1579 /* TODO Add error check if space will be sufficient */ 1580 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1581 1582 /* 1583 * Input Gather List 1584 */ 1585 i = 0; 1586 1587 /* Offset control word followed by iv */ 1588 1589 i = fill_sg_comp(gather_comp, i, offset_dma, 1590 OFF_CTRL_LEN + iv_len); 1591 1592 /* iv offset is 0 */ 1593 *offset_vaddr = offset_ctrl; 1594 1595 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1596 memcpy(iv_d, iv, 16); 1597 1598 /* input data */ 1599 size = inputlen - iv_len; 1600 if (size) { 1601 i = fill_sg_comp_from_iov(gather_comp, i, 1602 params->src_iov, 1603 0, &size, NULL, 0); 1604 if (unlikely(size)) { 1605 CPT_LOG_DP_ERR("Insufficient buffer space," 1606 " size %d needed", size); 1607 return; 1608 } 1609 } 1610 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1611 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1612 1613 /* 1614 * Output Scatter List 1615 */ 1616 1617 i = 0; 1618 scatter_comp = 1619 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1620 1621 if (flags == 0x1) { 1622 /* IV in SLIST only for EEA3 & UEA2 */ 1623 iv_len = 0; 1624 } 1625 1626 if (iv_len) { 1627 i = fill_sg_comp(scatter_comp, i, 1628 offset_dma + OFF_CTRL_LEN, iv_len); 1629 } 1630 1631 /* Add output data */ 1632 if (req_flags & VALID_MAC_BUF) { 1633 size = outputlen - iv_len - mac_len; 1634 if (size) { 1635 i = fill_sg_comp_from_iov(scatter_comp, i, 1636 params->dst_iov, 0, 1637 &size, NULL, 0); 1638 1639 if (unlikely(size)) { 1640 CPT_LOG_DP_ERR("Insufficient buffer space," 1641 " size %d needed", size); 1642 return; 1643 } 1644 } 1645 1646 /* mac data */ 1647 if (mac_len) { 1648 i = fill_sg_comp_from_buf(scatter_comp, i, 1649 ¶ms->mac_buf); 1650 } 1651 } else { 1652 /* Output including mac */ 1653 size = outputlen - iv_len; 1654 if (size) { 1655 i = fill_sg_comp_from_iov(scatter_comp, i, 1656 params->dst_iov, 0, 1657 &size, NULL, 0); 1658 1659 if (unlikely(size)) { 1660 CPT_LOG_DP_ERR("Insufficient buffer space," 1661 " size %d needed", size); 1662 return; 1663 } 1664 } 1665 } 1666 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1667 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1668 1669 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1670 1671 /* This is DPTR len incase of SG mode */ 1672 vq_cmd_w0.s.dlen = size; 1673 1674 m_vaddr = (uint8_t *)m_vaddr + size; 1675 m_dma += size; 1676 1677 /* cpt alternate completion address saved earlier */ 1678 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1679 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1680 rptr_dma = c_dma - 8; 1681 1682 req->ist.ei1 = dptr_dma; 1683 req->ist.ei2 = rptr_dma; 1684 } 1685 1686 /* 16 byte aligned cpt res address */ 1687 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1688 *req->completion_addr = COMPLETION_CODE_INIT; 1689 req->comp_baddr = c_dma; 1690 1691 /* Fill microcode part of instruction */ 1692 req->ist.ei0 = vq_cmd_w0.u64; 1693 1694 req->op = op; 1695 1696 *prep_req = req; 1697 return; 1698 } 1699 1700 static __rte_always_inline void 1701 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1702 uint64_t d_offs, 1703 uint64_t d_lens, 1704 fc_params_t *params, 1705 void *op, 1706 void **prep_req) 1707 { 1708 uint32_t size; 1709 int32_t inputlen = 0, outputlen; 1710 struct cpt_ctx *cpt_ctx; 1711 uint8_t snow3g, iv_len = 16; 1712 struct cpt_request_info *req; 1713 buf_ptr_t *buf_p; 1714 uint32_t encr_offset; 1715 uint32_t encr_data_len; 1716 int flags; 1717 void *m_vaddr, *c_vaddr; 1718 uint64_t m_dma, c_dma; 1719 uint64_t *offset_vaddr, offset_dma; 1720 uint32_t *iv_s, iv[4], j; 1721 vq_cmd_word0_t vq_cmd_w0; 1722 1723 buf_p = ¶ms->meta_buf; 1724 m_vaddr = buf_p->vaddr; 1725 m_dma = buf_p->dma_addr; 1726 1727 /* 1728 * Microcode expects offsets in bytes 1729 * TODO: Rounding off 1730 */ 1731 encr_offset = ENCR_OFFSET(d_offs) / 8; 1732 encr_data_len = ENCR_DLEN(d_lens); 1733 1734 cpt_ctx = params->ctx_buf.vaddr; 1735 flags = cpt_ctx->zsk_flags; 1736 snow3g = cpt_ctx->snow3g; 1737 /* 1738 * Save initial space that followed app data for completion code & 1739 * alternate completion code to fall in same cache line as app data 1740 */ 1741 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1742 m_dma += COMPLETION_CODE_SIZE; 1743 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1744 (uint8_t *)m_vaddr; 1745 1746 c_vaddr = (uint8_t *)m_vaddr + size; 1747 c_dma = m_dma + size; 1748 size += sizeof(cpt_res_s_t); 1749 1750 m_vaddr = (uint8_t *)m_vaddr + size; 1751 m_dma += size; 1752 1753 /* Reserve memory for cpt request info */ 1754 req = m_vaddr; 1755 1756 size = sizeof(struct cpt_request_info); 1757 m_vaddr = (uint8_t *)m_vaddr + size; 1758 m_dma += size; 1759 1760 vq_cmd_w0.u64 = 0; 1761 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1762 1763 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1764 1765 vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1766 (0 << 3) | (flags & 0x7)); 1767 1768 /* consider iv len */ 1769 encr_offset += iv_len; 1770 1771 inputlen = encr_offset + 1772 (RTE_ALIGN(encr_data_len, 8) / 8); 1773 outputlen = inputlen; 1774 1775 /* IV */ 1776 iv_s = params->iv_buf; 1777 if (snow3g) { 1778 /* 1779 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1780 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1781 */ 1782 1783 for (j = 0; j < 4; j++) 1784 iv[j] = iv_s[3 - j]; 1785 } else { 1786 /* ZUC doesn't need a swap */ 1787 for (j = 0; j < 4; j++) 1788 iv[j] = iv_s[j]; 1789 } 1790 1791 /* 1792 * GP op header, lengths are expected in bits. 1793 */ 1794 vq_cmd_w0.s.param1 = encr_data_len; 1795 1796 /* 1797 * In 83XX since we have a limitation of 1798 * IV & Offset control word not part of instruction 1799 * and need to be part of Data Buffer, we check if 1800 * head room is there and then only do the Direct mode processing 1801 */ 1802 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1803 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1804 void *dm_vaddr = params->bufs[0].vaddr; 1805 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1806 /* 1807 * This flag indicates that there is 24 bytes head room and 1808 * 8 bytes tail room available, so that we get to do 1809 * DIRECT MODE with limitation 1810 */ 1811 1812 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1813 OFF_CTRL_LEN - iv_len); 1814 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1815 1816 /* DPTR */ 1817 req->ist.ei1 = offset_dma; 1818 /* RPTR should just exclude offset control word */ 1819 req->ist.ei2 = dm_dma_addr - iv_len; 1820 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1821 + outputlen - iv_len); 1822 1823 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1824 1825 if (likely(iv_len)) { 1826 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1827 + OFF_CTRL_LEN); 1828 memcpy(iv_d, iv, 16); 1829 } 1830 1831 /* iv offset is 0 */ 1832 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1833 } else { 1834 uint32_t i, g_size_bytes, s_size_bytes; 1835 uint64_t dptr_dma, rptr_dma; 1836 sg_comp_t *gather_comp; 1837 sg_comp_t *scatter_comp; 1838 uint8_t *in_buffer; 1839 uint32_t *iv_d; 1840 1841 /* save space for offset and iv... */ 1842 offset_vaddr = m_vaddr; 1843 offset_dma = m_dma; 1844 1845 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1846 m_dma += OFF_CTRL_LEN + iv_len; 1847 1848 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; 1849 1850 /* DPTR has SG list */ 1851 in_buffer = m_vaddr; 1852 dptr_dma = m_dma; 1853 1854 ((uint16_t *)in_buffer)[0] = 0; 1855 ((uint16_t *)in_buffer)[1] = 0; 1856 1857 /* TODO Add error check if space will be sufficient */ 1858 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1859 1860 /* 1861 * Input Gather List 1862 */ 1863 i = 0; 1864 1865 /* Offset control word */ 1866 1867 /* iv offset is 0 */ 1868 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1869 1870 i = fill_sg_comp(gather_comp, i, offset_dma, 1871 OFF_CTRL_LEN + iv_len); 1872 1873 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1874 memcpy(iv_d, iv, 16); 1875 1876 /* Add input data */ 1877 size = inputlen - iv_len; 1878 if (size) { 1879 i = fill_sg_comp_from_iov(gather_comp, i, 1880 params->src_iov, 1881 0, &size, NULL, 0); 1882 if (unlikely(size)) { 1883 CPT_LOG_DP_ERR("Insufficient buffer space," 1884 " size %d needed", size); 1885 return; 1886 } 1887 } 1888 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1889 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1890 1891 /* 1892 * Output Scatter List 1893 */ 1894 1895 i = 0; 1896 scatter_comp = 1897 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1898 1899 /* IV */ 1900 i = fill_sg_comp(scatter_comp, i, 1901 offset_dma + OFF_CTRL_LEN, 1902 iv_len); 1903 1904 /* Add output data */ 1905 size = outputlen - iv_len; 1906 if (size) { 1907 i = fill_sg_comp_from_iov(scatter_comp, i, 1908 params->dst_iov, 0, 1909 &size, NULL, 0); 1910 1911 if (unlikely(size)) { 1912 CPT_LOG_DP_ERR("Insufficient buffer space," 1913 " size %d needed", size); 1914 return; 1915 } 1916 } 1917 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1918 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1919 1920 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1921 1922 /* This is DPTR len incase of SG mode */ 1923 vq_cmd_w0.s.dlen = size; 1924 1925 m_vaddr = (uint8_t *)m_vaddr + size; 1926 m_dma += size; 1927 1928 /* cpt alternate completion address saved earlier */ 1929 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1930 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1931 rptr_dma = c_dma - 8; 1932 1933 req->ist.ei1 = dptr_dma; 1934 req->ist.ei2 = rptr_dma; 1935 } 1936 1937 /* 16 byte aligned cpt res address */ 1938 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1939 *req->completion_addr = COMPLETION_CODE_INIT; 1940 req->comp_baddr = c_dma; 1941 1942 /* Fill microcode part of instruction */ 1943 req->ist.ei0 = vq_cmd_w0.u64; 1944 1945 req->op = op; 1946 1947 *prep_req = req; 1948 return; 1949 } 1950 1951 static __rte_always_inline void 1952 cpt_kasumi_enc_prep(uint32_t req_flags, 1953 uint64_t d_offs, 1954 uint64_t d_lens, 1955 fc_params_t *params, 1956 void *op, 1957 void **prep_req) 1958 { 1959 uint32_t size; 1960 int32_t inputlen = 0, outputlen = 0; 1961 struct cpt_ctx *cpt_ctx; 1962 uint32_t mac_len = 0; 1963 uint8_t i = 0; 1964 struct cpt_request_info *req; 1965 buf_ptr_t *buf_p; 1966 uint32_t encr_offset, auth_offset; 1967 uint32_t encr_data_len, auth_data_len; 1968 int flags; 1969 uint8_t *iv_s, *iv_d, iv_len = 8; 1970 uint8_t dir = 0; 1971 void *m_vaddr, *c_vaddr; 1972 uint64_t m_dma, c_dma; 1973 uint64_t *offset_vaddr, offset_dma; 1974 vq_cmd_word0_t vq_cmd_w0; 1975 uint8_t *in_buffer; 1976 uint32_t g_size_bytes, s_size_bytes; 1977 uint64_t dptr_dma, rptr_dma; 1978 sg_comp_t *gather_comp; 1979 sg_comp_t *scatter_comp; 1980 1981 buf_p = ¶ms->meta_buf; 1982 m_vaddr = buf_p->vaddr; 1983 m_dma = buf_p->dma_addr; 1984 1985 encr_offset = ENCR_OFFSET(d_offs) / 8; 1986 auth_offset = AUTH_OFFSET(d_offs) / 8; 1987 encr_data_len = ENCR_DLEN(d_lens); 1988 auth_data_len = AUTH_DLEN(d_lens); 1989 1990 cpt_ctx = params->ctx_buf.vaddr; 1991 flags = cpt_ctx->zsk_flags; 1992 mac_len = cpt_ctx->mac_len; 1993 1994 if (flags == 0x0) 1995 iv_s = params->iv_buf; 1996 else 1997 iv_s = params->auth_iv_buf; 1998 1999 dir = iv_s[8] & 0x1; 2000 2001 /* 2002 * Save initial space that followed app data for completion code & 2003 * alternate completion code to fall in same cache line as app data 2004 */ 2005 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2006 m_dma += COMPLETION_CODE_SIZE; 2007 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2008 (uint8_t *)m_vaddr; 2009 2010 c_vaddr = (uint8_t *)m_vaddr + size; 2011 c_dma = m_dma + size; 2012 size += sizeof(cpt_res_s_t); 2013 2014 m_vaddr = (uint8_t *)m_vaddr + size; 2015 m_dma += size; 2016 2017 /* Reserve memory for cpt request info */ 2018 req = m_vaddr; 2019 2020 size = sizeof(struct cpt_request_info); 2021 m_vaddr = (uint8_t *)m_vaddr + size; 2022 m_dma += size; 2023 2024 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2025 2026 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2027 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2028 (dir << 4) | (0 << 3) | (flags & 0x7)); 2029 2030 /* 2031 * GP op header, lengths are expected in bits. 2032 */ 2033 vq_cmd_w0.s.param1 = encr_data_len; 2034 vq_cmd_w0.s.param2 = auth_data_len; 2035 2036 /* consider iv len */ 2037 if (flags == 0x0) { 2038 encr_offset += iv_len; 2039 auth_offset += iv_len; 2040 } 2041 2042 /* save space for offset ctrl and iv */ 2043 offset_vaddr = m_vaddr; 2044 offset_dma = m_dma; 2045 2046 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2047 m_dma += OFF_CTRL_LEN + iv_len; 2048 2049 /* DPTR has SG list */ 2050 in_buffer = m_vaddr; 2051 dptr_dma = m_dma; 2052 2053 ((uint16_t *)in_buffer)[0] = 0; 2054 ((uint16_t *)in_buffer)[1] = 0; 2055 2056 /* TODO Add error check if space will be sufficient */ 2057 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2058 2059 /* 2060 * Input Gather List 2061 */ 2062 i = 0; 2063 2064 /* Offset control word followed by iv */ 2065 2066 if (flags == 0x0) { 2067 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2068 outputlen = inputlen; 2069 /* iv offset is 0 */ 2070 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2071 } else { 2072 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2073 outputlen = mac_len; 2074 /* iv offset is 0 */ 2075 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2076 } 2077 2078 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2079 2080 /* IV */ 2081 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2082 memcpy(iv_d, iv_s, iv_len); 2083 2084 /* input data */ 2085 size = inputlen - iv_len; 2086 if (size) { 2087 i = fill_sg_comp_from_iov(gather_comp, i, 2088 params->src_iov, 0, 2089 &size, NULL, 0); 2090 2091 if (unlikely(size)) { 2092 CPT_LOG_DP_ERR("Insufficient buffer space," 2093 " size %d needed", size); 2094 return; 2095 } 2096 } 2097 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2098 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2099 2100 /* 2101 * Output Scatter List 2102 */ 2103 2104 i = 0; 2105 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2106 2107 if (flags == 0x1) { 2108 /* IV in SLIST only for F8 */ 2109 iv_len = 0; 2110 } 2111 2112 /* IV */ 2113 if (iv_len) { 2114 i = fill_sg_comp(scatter_comp, i, 2115 offset_dma + OFF_CTRL_LEN, 2116 iv_len); 2117 } 2118 2119 /* Add output data */ 2120 if (req_flags & VALID_MAC_BUF) { 2121 size = outputlen - iv_len - mac_len; 2122 if (size) { 2123 i = fill_sg_comp_from_iov(scatter_comp, i, 2124 params->dst_iov, 0, 2125 &size, NULL, 0); 2126 2127 if (unlikely(size)) { 2128 CPT_LOG_DP_ERR("Insufficient buffer space," 2129 " size %d needed", size); 2130 return; 2131 } 2132 } 2133 2134 /* mac data */ 2135 if (mac_len) { 2136 i = fill_sg_comp_from_buf(scatter_comp, i, 2137 ¶ms->mac_buf); 2138 } 2139 } else { 2140 /* Output including mac */ 2141 size = outputlen - iv_len; 2142 if (size) { 2143 i = fill_sg_comp_from_iov(scatter_comp, i, 2144 params->dst_iov, 0, 2145 &size, NULL, 0); 2146 2147 if (unlikely(size)) { 2148 CPT_LOG_DP_ERR("Insufficient buffer space," 2149 " size %d needed", size); 2150 return; 2151 } 2152 } 2153 } 2154 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2155 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2156 2157 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2158 2159 /* This is DPTR len incase of SG mode */ 2160 vq_cmd_w0.s.dlen = size; 2161 2162 m_vaddr = (uint8_t *)m_vaddr + size; 2163 m_dma += size; 2164 2165 /* cpt alternate completion address saved earlier */ 2166 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2167 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2168 rptr_dma = c_dma - 8; 2169 2170 req->ist.ei1 = dptr_dma; 2171 req->ist.ei2 = rptr_dma; 2172 2173 /* 16 byte aligned cpt res address */ 2174 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2175 *req->completion_addr = COMPLETION_CODE_INIT; 2176 req->comp_baddr = c_dma; 2177 2178 /* Fill microcode part of instruction */ 2179 req->ist.ei0 = vq_cmd_w0.u64; 2180 2181 req->op = op; 2182 2183 *prep_req = req; 2184 return; 2185 } 2186 2187 static __rte_always_inline void 2188 cpt_kasumi_dec_prep(uint64_t d_offs, 2189 uint64_t d_lens, 2190 fc_params_t *params, 2191 void *op, 2192 void **prep_req) 2193 { 2194 uint32_t size; 2195 int32_t inputlen = 0, outputlen; 2196 struct cpt_ctx *cpt_ctx; 2197 uint8_t i = 0, iv_len = 8; 2198 struct cpt_request_info *req; 2199 buf_ptr_t *buf_p; 2200 uint32_t encr_offset; 2201 uint32_t encr_data_len; 2202 int flags; 2203 uint8_t dir = 0; 2204 void *m_vaddr, *c_vaddr; 2205 uint64_t m_dma, c_dma; 2206 uint64_t *offset_vaddr, offset_dma; 2207 vq_cmd_word0_t vq_cmd_w0; 2208 uint8_t *in_buffer; 2209 uint32_t g_size_bytes, s_size_bytes; 2210 uint64_t dptr_dma, rptr_dma; 2211 sg_comp_t *gather_comp; 2212 sg_comp_t *scatter_comp; 2213 2214 buf_p = ¶ms->meta_buf; 2215 m_vaddr = buf_p->vaddr; 2216 m_dma = buf_p->dma_addr; 2217 2218 encr_offset = ENCR_OFFSET(d_offs) / 8; 2219 encr_data_len = ENCR_DLEN(d_lens); 2220 2221 cpt_ctx = params->ctx_buf.vaddr; 2222 flags = cpt_ctx->zsk_flags; 2223 /* 2224 * Save initial space that followed app data for completion code & 2225 * alternate completion code to fall in same cache line as app data 2226 */ 2227 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2228 m_dma += COMPLETION_CODE_SIZE; 2229 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2230 (uint8_t *)m_vaddr; 2231 2232 c_vaddr = (uint8_t *)m_vaddr + size; 2233 c_dma = m_dma + size; 2234 size += sizeof(cpt_res_s_t); 2235 2236 m_vaddr = (uint8_t *)m_vaddr + size; 2237 m_dma += size; 2238 2239 /* Reserve memory for cpt request info */ 2240 req = m_vaddr; 2241 2242 size = sizeof(struct cpt_request_info); 2243 m_vaddr = (uint8_t *)m_vaddr + size; 2244 m_dma += size; 2245 2246 vq_cmd_w0.u64 = 0; 2247 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2248 2249 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2250 vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2251 (dir << 4) | (0 << 3) | (flags & 0x7)); 2252 2253 /* 2254 * GP op header, lengths are expected in bits. 2255 */ 2256 vq_cmd_w0.s.param1 = encr_data_len; 2257 2258 /* consider iv len */ 2259 encr_offset += iv_len; 2260 2261 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2262 outputlen = inputlen; 2263 2264 /* save space for offset ctrl & iv */ 2265 offset_vaddr = m_vaddr; 2266 offset_dma = m_dma; 2267 2268 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2269 m_dma += OFF_CTRL_LEN + iv_len; 2270 2271 /* DPTR has SG list */ 2272 in_buffer = m_vaddr; 2273 dptr_dma = m_dma; 2274 2275 ((uint16_t *)in_buffer)[0] = 0; 2276 ((uint16_t *)in_buffer)[1] = 0; 2277 2278 /* TODO Add error check if space will be sufficient */ 2279 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2280 2281 /* 2282 * Input Gather List 2283 */ 2284 i = 0; 2285 2286 /* Offset control word followed by iv */ 2287 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2288 2289 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2290 2291 /* IV */ 2292 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2293 params->iv_buf, iv_len); 2294 2295 /* Add input data */ 2296 size = inputlen - iv_len; 2297 if (size) { 2298 i = fill_sg_comp_from_iov(gather_comp, i, 2299 params->src_iov, 2300 0, &size, NULL, 0); 2301 if (unlikely(size)) { 2302 CPT_LOG_DP_ERR("Insufficient buffer space," 2303 " size %d needed", size); 2304 return; 2305 } 2306 } 2307 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2308 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2309 2310 /* 2311 * Output Scatter List 2312 */ 2313 2314 i = 0; 2315 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2316 2317 /* IV */ 2318 i = fill_sg_comp(scatter_comp, i, 2319 offset_dma + OFF_CTRL_LEN, 2320 iv_len); 2321 2322 /* Add output data */ 2323 size = outputlen - iv_len; 2324 if (size) { 2325 i = fill_sg_comp_from_iov(scatter_comp, i, 2326 params->dst_iov, 0, 2327 &size, NULL, 0); 2328 if (unlikely(size)) { 2329 CPT_LOG_DP_ERR("Insufficient buffer space," 2330 " size %d needed", size); 2331 return; 2332 } 2333 } 2334 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2335 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2336 2337 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2338 2339 /* This is DPTR len incase of SG mode */ 2340 vq_cmd_w0.s.dlen = size; 2341 2342 m_vaddr = (uint8_t *)m_vaddr + size; 2343 m_dma += size; 2344 2345 /* cpt alternate completion address saved earlier */ 2346 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2347 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2348 rptr_dma = c_dma - 8; 2349 2350 req->ist.ei1 = dptr_dma; 2351 req->ist.ei2 = rptr_dma; 2352 2353 /* 16 byte aligned cpt res address */ 2354 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2355 *req->completion_addr = COMPLETION_CODE_INIT; 2356 req->comp_baddr = c_dma; 2357 2358 /* Fill microcode part of instruction */ 2359 req->ist.ei0 = vq_cmd_w0.u64; 2360 2361 req->op = op; 2362 2363 *prep_req = req; 2364 return; 2365 } 2366 2367 static __rte_always_inline void * 2368 cpt_fc_dec_hmac_prep(uint32_t flags, 2369 uint64_t d_offs, 2370 uint64_t d_lens, 2371 fc_params_t *fc_params, 2372 void *op) 2373 { 2374 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2375 uint8_t fc_type; 2376 void *prep_req = NULL; 2377 2378 fc_type = ctx->fc_type; 2379 2380 if (likely(fc_type == FC_GEN)) { 2381 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2382 &prep_req); 2383 } else if (fc_type == ZUC_SNOW3G) { 2384 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2385 &prep_req); 2386 } else if (fc_type == KASUMI) { 2387 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2388 } 2389 2390 /* 2391 * For AUTH_ONLY case, 2392 * MC only supports digest generation and verification 2393 * should be done in software by memcmp() 2394 */ 2395 2396 return prep_req; 2397 } 2398 2399 static __rte_always_inline void *__rte_hot 2400 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2401 fc_params_t *fc_params, void *op) 2402 { 2403 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2404 uint8_t fc_type; 2405 void *prep_req = NULL; 2406 2407 fc_type = ctx->fc_type; 2408 2409 /* Common api for rest of the ops */ 2410 if (likely(fc_type == FC_GEN)) { 2411 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2412 &prep_req); 2413 } else if (fc_type == ZUC_SNOW3G) { 2414 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2415 &prep_req); 2416 } else if (fc_type == KASUMI) { 2417 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2418 &prep_req); 2419 } else if (fc_type == HASH_HMAC) { 2420 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2421 } 2422 2423 return prep_req; 2424 } 2425 2426 static __rte_always_inline int 2427 cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type, 2428 const uint8_t *key, uint16_t key_len, uint16_t mac_len) 2429 { 2430 mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; 2431 mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; 2432 mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; 2433 2434 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2435 uint32_t keyx[4]; 2436 2437 if (key_len != 16) 2438 return -1; 2439 /* No support for AEAD yet */ 2440 if (cpt_ctx->enc_cipher) 2441 return -1; 2442 /* For ZUC/SNOW3G/Kasumi */ 2443 switch (type) { 2444 case SNOW3G_UIA2: 2445 cpt_ctx->snow3g = 1; 2446 gen_key_snow3g(key, keyx); 2447 memcpy(zs_ctx->ci_key, keyx, key_len); 2448 cpt_ctx->fc_type = ZUC_SNOW3G; 2449 cpt_ctx->zsk_flags = 0x1; 2450 break; 2451 case ZUC_EIA3: 2452 cpt_ctx->snow3g = 0; 2453 memcpy(zs_ctx->ci_key, key, key_len); 2454 memcpy(zs_ctx->zuc_const, zuc_d, 32); 2455 cpt_ctx->fc_type = ZUC_SNOW3G; 2456 cpt_ctx->zsk_flags = 0x1; 2457 break; 2458 case KASUMI_F9_ECB: 2459 /* Kasumi ECB mode */ 2460 cpt_ctx->k_ecb = 1; 2461 memcpy(k_ctx->ci_key, key, key_len); 2462 cpt_ctx->fc_type = KASUMI; 2463 cpt_ctx->zsk_flags = 0x1; 2464 break; 2465 case KASUMI_F9_CBC: 2466 memcpy(k_ctx->ci_key, key, key_len); 2467 cpt_ctx->fc_type = KASUMI; 2468 cpt_ctx->zsk_flags = 0x1; 2469 break; 2470 default: 2471 return -1; 2472 } 2473 cpt_ctx->mac_len = 4; 2474 cpt_ctx->hash_type = type; 2475 return 0; 2476 } 2477 2478 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2479 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2480 cpt_ctx->fc_type = HASH_HMAC; 2481 } 2482 2483 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2484 return -1; 2485 2486 /* For GMAC auth, cipher must be NULL */ 2487 if (type == GMAC_TYPE) 2488 fctx->enc.enc_cipher = 0; 2489 2490 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2491 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2492 2493 if (key_len) { 2494 cpt_ctx->hmac = 1; 2495 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2496 memcpy(cpt_ctx->auth_key, key, key_len); 2497 cpt_ctx->auth_key_len = key_len; 2498 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2499 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2500 2501 if (key_len <= 64) 2502 memcpy(fctx->hmac.opad, key, key_len); 2503 fctx->enc.auth_input_type = 1; 2504 } 2505 return 0; 2506 } 2507 2508 static __rte_always_inline int 2509 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2510 struct cpt_sess_misc *sess) 2511 { 2512 struct rte_crypto_aead_xform *aead_form; 2513 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2514 auth_type_t auth_type = 0; /* NULL Auth type */ 2515 uint32_t cipher_key_len = 0; 2516 uint8_t aes_gcm = 0; 2517 aead_form = &xform->aead; 2518 void *ctx = SESS_PRIV(sess); 2519 2520 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 2521 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2522 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2523 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) { 2524 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2525 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2526 } else { 2527 CPT_LOG_DP_ERR("Unknown aead operation\n"); 2528 return -1; 2529 } 2530 switch (aead_form->algo) { 2531 case RTE_CRYPTO_AEAD_AES_GCM: 2532 enc_type = AES_GCM; 2533 cipher_key_len = 16; 2534 aes_gcm = 1; 2535 break; 2536 case RTE_CRYPTO_AEAD_AES_CCM: 2537 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2538 aead_form->algo); 2539 return -1; 2540 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: 2541 enc_type = CHACHA20; 2542 auth_type = POLY1305; 2543 cipher_key_len = 32; 2544 sess->chacha_poly = 1; 2545 break; 2546 default: 2547 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2548 aead_form->algo); 2549 return -1; 2550 } 2551 if (aead_form->key.length < cipher_key_len) { 2552 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2553 (unsigned int long)aead_form->key.length); 2554 return -1; 2555 } 2556 sess->zsk_flag = 0; 2557 sess->aes_gcm = aes_gcm; 2558 sess->mac_len = aead_form->digest_length; 2559 sess->iv_offset = aead_form->iv.offset; 2560 sess->iv_length = aead_form->iv.length; 2561 sess->aad_length = aead_form->aad_length; 2562 2563 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2564 aead_form->key.length, NULL))) 2565 return -1; 2566 2567 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2568 aead_form->digest_length))) 2569 return -1; 2570 2571 return 0; 2572 } 2573 2574 static __rte_always_inline int 2575 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2576 struct cpt_sess_misc *sess) 2577 { 2578 struct rte_crypto_cipher_xform *c_form; 2579 struct cpt_ctx *ctx = SESS_PRIV(sess); 2580 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2581 uint32_t cipher_key_len = 0; 2582 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2583 2584 c_form = &xform->cipher; 2585 2586 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2587 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2588 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 2589 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2590 if (xform->next != NULL && 2591 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2592 /* Perform decryption followed by auth verify */ 2593 ctx->dec_auth = 1; 2594 } 2595 } else { 2596 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2597 return -1; 2598 } 2599 2600 switch (c_form->algo) { 2601 case RTE_CRYPTO_CIPHER_AES_CBC: 2602 enc_type = AES_CBC; 2603 cipher_key_len = 16; 2604 break; 2605 case RTE_CRYPTO_CIPHER_3DES_CBC: 2606 enc_type = DES3_CBC; 2607 cipher_key_len = 24; 2608 break; 2609 case RTE_CRYPTO_CIPHER_DES_CBC: 2610 /* DES is implemented using 3DES in hardware */ 2611 enc_type = DES3_CBC; 2612 cipher_key_len = 8; 2613 break; 2614 case RTE_CRYPTO_CIPHER_AES_CTR: 2615 enc_type = AES_CTR; 2616 cipher_key_len = 16; 2617 aes_ctr = 1; 2618 break; 2619 case RTE_CRYPTO_CIPHER_NULL: 2620 enc_type = 0; 2621 is_null = 1; 2622 break; 2623 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2624 enc_type = KASUMI_F8_ECB; 2625 cipher_key_len = 16; 2626 zsk_flag = K_F8; 2627 break; 2628 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2629 enc_type = SNOW3G_UEA2; 2630 cipher_key_len = 16; 2631 zsk_flag = ZS_EA; 2632 break; 2633 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2634 enc_type = ZUC_EEA3; 2635 cipher_key_len = 16; 2636 zsk_flag = ZS_EA; 2637 break; 2638 case RTE_CRYPTO_CIPHER_AES_XTS: 2639 enc_type = AES_XTS; 2640 cipher_key_len = 16; 2641 break; 2642 case RTE_CRYPTO_CIPHER_3DES_ECB: 2643 enc_type = DES3_ECB; 2644 cipher_key_len = 24; 2645 break; 2646 case RTE_CRYPTO_CIPHER_AES_ECB: 2647 enc_type = AES_ECB; 2648 cipher_key_len = 16; 2649 break; 2650 case RTE_CRYPTO_CIPHER_3DES_CTR: 2651 case RTE_CRYPTO_CIPHER_AES_F8: 2652 case RTE_CRYPTO_CIPHER_ARC4: 2653 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2654 c_form->algo); 2655 return -1; 2656 default: 2657 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2658 c_form->algo); 2659 return -1; 2660 } 2661 2662 if (c_form->key.length < cipher_key_len) { 2663 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2664 (unsigned long) c_form->key.length); 2665 return -1; 2666 } 2667 2668 sess->zsk_flag = zsk_flag; 2669 sess->aes_gcm = 0; 2670 sess->aes_ctr = aes_ctr; 2671 sess->iv_offset = c_form->iv.offset; 2672 sess->iv_length = c_form->iv.length; 2673 sess->is_null = is_null; 2674 2675 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2676 c_form->key.data, c_form->key.length, NULL))) 2677 return -1; 2678 2679 return 0; 2680 } 2681 2682 static __rte_always_inline int 2683 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2684 struct cpt_sess_misc *sess) 2685 { 2686 struct cpt_ctx *ctx = SESS_PRIV(sess); 2687 struct rte_crypto_auth_xform *a_form; 2688 auth_type_t auth_type = 0; /* NULL Auth type */ 2689 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2690 2691 if (xform->next != NULL && 2692 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2693 xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 2694 /* Perform auth followed by encryption */ 2695 ctx->auth_enc = 1; 2696 } 2697 2698 a_form = &xform->auth; 2699 2700 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2701 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2702 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2703 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2704 else { 2705 CPT_LOG_DP_ERR("Unknown auth operation"); 2706 return -1; 2707 } 2708 2709 switch (a_form->algo) { 2710 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2711 /* Fall through */ 2712 case RTE_CRYPTO_AUTH_SHA1: 2713 auth_type = SHA1_TYPE; 2714 break; 2715 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2716 case RTE_CRYPTO_AUTH_SHA256: 2717 auth_type = SHA2_SHA256; 2718 break; 2719 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2720 case RTE_CRYPTO_AUTH_SHA512: 2721 auth_type = SHA2_SHA512; 2722 break; 2723 case RTE_CRYPTO_AUTH_AES_GMAC: 2724 auth_type = GMAC_TYPE; 2725 aes_gcm = 1; 2726 break; 2727 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2728 case RTE_CRYPTO_AUTH_SHA224: 2729 auth_type = SHA2_SHA224; 2730 break; 2731 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2732 case RTE_CRYPTO_AUTH_SHA384: 2733 auth_type = SHA2_SHA384; 2734 break; 2735 case RTE_CRYPTO_AUTH_MD5_HMAC: 2736 case RTE_CRYPTO_AUTH_MD5: 2737 auth_type = MD5_TYPE; 2738 break; 2739 case RTE_CRYPTO_AUTH_KASUMI_F9: 2740 auth_type = KASUMI_F9_ECB; 2741 /* 2742 * Indicate that direction needs to be taken out 2743 * from end of src 2744 */ 2745 zsk_flag = K_F9; 2746 break; 2747 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2748 auth_type = SNOW3G_UIA2; 2749 zsk_flag = ZS_IA; 2750 break; 2751 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2752 auth_type = ZUC_EIA3; 2753 zsk_flag = ZS_IA; 2754 break; 2755 case RTE_CRYPTO_AUTH_NULL: 2756 auth_type = 0; 2757 is_null = 1; 2758 break; 2759 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2760 case RTE_CRYPTO_AUTH_AES_CMAC: 2761 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2762 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2763 a_form->algo); 2764 return -1; 2765 default: 2766 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2767 a_form->algo); 2768 return -1; 2769 } 2770 2771 sess->zsk_flag = zsk_flag; 2772 sess->aes_gcm = aes_gcm; 2773 sess->mac_len = a_form->digest_length; 2774 sess->is_null = is_null; 2775 if (zsk_flag) { 2776 sess->auth_iv_offset = a_form->iv.offset; 2777 sess->auth_iv_length = a_form->iv.length; 2778 } 2779 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2780 a_form->key.data, a_form->key.length, 2781 a_form->digest_length))) 2782 return -1; 2783 2784 return 0; 2785 } 2786 2787 static __rte_always_inline int 2788 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2789 struct cpt_sess_misc *sess) 2790 { 2791 struct rte_crypto_auth_xform *a_form; 2792 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2793 auth_type_t auth_type = 0; /* NULL Auth type */ 2794 void *ctx = SESS_PRIV(sess); 2795 2796 a_form = &xform->auth; 2797 2798 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2799 sess->cpt_op |= CPT_OP_ENCODE; 2800 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2801 sess->cpt_op |= CPT_OP_DECODE; 2802 else { 2803 CPT_LOG_DP_ERR("Unknown auth operation"); 2804 return -1; 2805 } 2806 2807 switch (a_form->algo) { 2808 case RTE_CRYPTO_AUTH_AES_GMAC: 2809 enc_type = AES_GCM; 2810 auth_type = GMAC_TYPE; 2811 break; 2812 default: 2813 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2814 a_form->algo); 2815 return -1; 2816 } 2817 2818 sess->zsk_flag = 0; 2819 sess->aes_gcm = 0; 2820 sess->is_gmac = 1; 2821 sess->iv_offset = a_form->iv.offset; 2822 sess->iv_length = a_form->iv.length; 2823 sess->mac_len = a_form->digest_length; 2824 2825 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2826 a_form->key.length, NULL))) 2827 return -1; 2828 2829 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2830 a_form->digest_length))) 2831 return -1; 2832 2833 return 0; 2834 } 2835 2836 static __rte_always_inline void * 2837 alloc_op_meta(struct rte_mbuf *m_src, 2838 buf_ptr_t *buf, 2839 int32_t len, 2840 struct rte_mempool *cpt_meta_pool) 2841 { 2842 uint8_t *mdata; 2843 2844 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2845 if (likely(m_src && (m_src->nb_segs == 1))) { 2846 int32_t tailroom; 2847 phys_addr_t mphys; 2848 2849 /* Check if tailroom is sufficient to hold meta data */ 2850 tailroom = rte_pktmbuf_tailroom(m_src); 2851 if (likely(tailroom > len + 8)) { 2852 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2853 mphys = m_src->buf_iova + m_src->buf_len; 2854 mdata -= len; 2855 mphys -= len; 2856 buf->vaddr = mdata; 2857 buf->dma_addr = mphys; 2858 buf->size = len; 2859 /* Indicate that this is a mbuf allocated mdata */ 2860 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2861 return mdata; 2862 } 2863 } 2864 #else 2865 RTE_SET_USED(m_src); 2866 #endif 2867 2868 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2869 return NULL; 2870 2871 buf->vaddr = mdata; 2872 buf->dma_addr = rte_mempool_virt2iova(mdata); 2873 buf->size = len; 2874 2875 return mdata; 2876 } 2877 2878 /** 2879 * cpt_free_metabuf - free metabuf to mempool. 2880 * @param instance: pointer to instance. 2881 * @param objp: pointer to the metabuf. 2882 */ 2883 static __rte_always_inline void 2884 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2885 { 2886 bool nofree = ((uintptr_t)mdata & 1ull); 2887 2888 if (likely(nofree)) 2889 return; 2890 rte_mempool_put(cpt_meta_pool, mdata); 2891 } 2892 2893 static __rte_always_inline uint32_t 2894 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2895 iov_ptr_t *iovec, uint32_t start_offset) 2896 { 2897 uint16_t index = 0; 2898 void *seg_data = NULL; 2899 phys_addr_t seg_phys; 2900 int32_t seg_size = 0; 2901 2902 if (!pkt) { 2903 iovec->buf_cnt = 0; 2904 return 0; 2905 } 2906 2907 if (!start_offset) { 2908 seg_data = rte_pktmbuf_mtod(pkt, void *); 2909 seg_phys = rte_pktmbuf_iova(pkt); 2910 seg_size = pkt->data_len; 2911 } else { 2912 while (start_offset >= pkt->data_len) { 2913 start_offset -= pkt->data_len; 2914 pkt = pkt->next; 2915 } 2916 2917 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2918 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset); 2919 seg_size = pkt->data_len - start_offset; 2920 if (!seg_size) 2921 return 1; 2922 } 2923 2924 /* first seg */ 2925 iovec->bufs[index].vaddr = seg_data; 2926 iovec->bufs[index].dma_addr = seg_phys; 2927 iovec->bufs[index].size = seg_size; 2928 index++; 2929 pkt = pkt->next; 2930 2931 while (unlikely(pkt != NULL)) { 2932 seg_data = rte_pktmbuf_mtod(pkt, void *); 2933 seg_phys = rte_pktmbuf_iova(pkt); 2934 seg_size = pkt->data_len; 2935 if (!seg_size) 2936 break; 2937 2938 iovec->bufs[index].vaddr = seg_data; 2939 iovec->bufs[index].dma_addr = seg_phys; 2940 iovec->bufs[index].size = seg_size; 2941 2942 index++; 2943 2944 pkt = pkt->next; 2945 } 2946 2947 iovec->buf_cnt = index; 2948 return 0; 2949 } 2950 2951 static __rte_always_inline uint32_t 2952 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2953 fc_params_t *param, 2954 uint32_t *flags) 2955 { 2956 uint16_t index = 0; 2957 void *seg_data = NULL; 2958 phys_addr_t seg_phys; 2959 uint32_t seg_size = 0; 2960 iov_ptr_t *iovec; 2961 2962 seg_data = rte_pktmbuf_mtod(pkt, void *); 2963 seg_phys = rte_pktmbuf_iova(pkt); 2964 seg_size = pkt->data_len; 2965 2966 /* first seg */ 2967 if (likely(!pkt->next)) { 2968 uint32_t headroom, tailroom; 2969 2970 *flags |= SINGLE_BUF_INPLACE; 2971 headroom = rte_pktmbuf_headroom(pkt); 2972 tailroom = rte_pktmbuf_tailroom(pkt); 2973 if (likely((headroom >= 24) && 2974 (tailroom >= 8))) { 2975 /* In 83XX this is prerequivisit for Direct mode */ 2976 *flags |= SINGLE_BUF_HEADTAILROOM; 2977 } 2978 param->bufs[0].vaddr = seg_data; 2979 param->bufs[0].dma_addr = seg_phys; 2980 param->bufs[0].size = seg_size; 2981 return 0; 2982 } 2983 iovec = param->src_iov; 2984 iovec->bufs[index].vaddr = seg_data; 2985 iovec->bufs[index].dma_addr = seg_phys; 2986 iovec->bufs[index].size = seg_size; 2987 index++; 2988 pkt = pkt->next; 2989 2990 while (unlikely(pkt != NULL)) { 2991 seg_data = rte_pktmbuf_mtod(pkt, void *); 2992 seg_phys = rte_pktmbuf_iova(pkt); 2993 seg_size = pkt->data_len; 2994 2995 if (!seg_size) 2996 break; 2997 2998 iovec->bufs[index].vaddr = seg_data; 2999 iovec->bufs[index].dma_addr = seg_phys; 3000 iovec->bufs[index].size = seg_size; 3001 3002 index++; 3003 3004 pkt = pkt->next; 3005 } 3006 3007 iovec->buf_cnt = index; 3008 return 0; 3009 } 3010 3011 static __rte_always_inline int 3012 fill_fc_params(struct rte_crypto_op *cop, 3013 struct cpt_sess_misc *sess_misc, 3014 struct cpt_qp_meta_info *m_info, 3015 void **mdata_ptr, 3016 void **prep_req) 3017 { 3018 uint32_t space = 0; 3019 struct rte_crypto_sym_op *sym_op = cop->sym; 3020 struct cpt_ctx *ctx = SESS_PRIV(sess_misc); 3021 void *mdata = NULL; 3022 uintptr_t *op; 3023 uint32_t mc_hash_off; 3024 uint32_t flags = 0; 3025 uint64_t d_offs, d_lens; 3026 struct rte_mbuf *m_src, *m_dst; 3027 uint8_t cpt_op = sess_misc->cpt_op; 3028 #ifdef CPT_ALWAYS_USE_SG_MODE 3029 uint8_t inplace = 0; 3030 #else 3031 uint8_t inplace = 1; 3032 #endif 3033 fc_params_t fc_params; 3034 char src[SRC_IOV_SIZE]; 3035 char dst[SRC_IOV_SIZE]; 3036 uint32_t iv_buf[4]; 3037 int ret; 3038 3039 if (likely(sess_misc->iv_length)) { 3040 flags |= VALID_IV_BUF; 3041 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3042 uint8_t *, sess_misc->iv_offset); 3043 if (sess_misc->aes_ctr && 3044 unlikely(sess_misc->iv_length != 16)) { 3045 memcpy((uint8_t *)iv_buf, 3046 rte_crypto_op_ctod_offset(cop, 3047 uint8_t *, sess_misc->iv_offset), 12); 3048 iv_buf[3] = rte_cpu_to_be_32(0x1); 3049 fc_params.iv_buf = iv_buf; 3050 } 3051 } 3052 3053 if (sess_misc->zsk_flag) { 3054 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3055 uint8_t *, 3056 sess_misc->auth_iv_offset); 3057 if (sess_misc->zsk_flag != ZS_EA) 3058 inplace = 0; 3059 } 3060 m_src = sym_op->m_src; 3061 m_dst = sym_op->m_dst; 3062 3063 if (sess_misc->aes_gcm || sess_misc->chacha_poly) { 3064 uint8_t *salt; 3065 uint8_t *aad_data; 3066 uint16_t aad_len; 3067 3068 d_offs = sym_op->aead.data.offset; 3069 d_lens = sym_op->aead.data.length; 3070 mc_hash_off = sym_op->aead.data.offset + 3071 sym_op->aead.data.length; 3072 3073 aad_data = sym_op->aead.aad.data; 3074 aad_len = sess_misc->aad_length; 3075 if (likely((aad_data + aad_len) == 3076 rte_pktmbuf_mtod_offset(m_src, 3077 uint8_t *, 3078 sym_op->aead.data.offset))) { 3079 d_offs = (d_offs - aad_len) | (d_offs << 16); 3080 d_lens = (d_lens + aad_len) | (d_lens << 32); 3081 } else { 3082 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3083 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3084 fc_params.aad_buf.size = aad_len; 3085 flags |= VALID_AAD_BUF; 3086 inplace = 0; 3087 d_offs = d_offs << 16; 3088 d_lens = d_lens << 32; 3089 } 3090 3091 salt = fc_params.iv_buf; 3092 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3093 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3094 sess_misc->salt = *(uint32_t *)salt; 3095 } 3096 fc_params.iv_buf = salt + 4; 3097 if (likely(sess_misc->mac_len)) { 3098 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3099 m_src; 3100 3101 if (!m) 3102 m = m_src; 3103 3104 /* hmac immediately following data is best case */ 3105 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3106 mc_hash_off != 3107 (uint8_t *)sym_op->aead.digest.data)) { 3108 flags |= VALID_MAC_BUF; 3109 fc_params.mac_buf.size = sess_misc->mac_len; 3110 fc_params.mac_buf.vaddr = 3111 sym_op->aead.digest.data; 3112 fc_params.mac_buf.dma_addr = 3113 sym_op->aead.digest.phys_addr; 3114 inplace = 0; 3115 } 3116 } 3117 } else { 3118 d_offs = sym_op->cipher.data.offset; 3119 d_lens = sym_op->cipher.data.length; 3120 mc_hash_off = sym_op->cipher.data.offset + 3121 sym_op->cipher.data.length; 3122 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3123 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3124 3125 if (mc_hash_off < (sym_op->auth.data.offset + 3126 sym_op->auth.data.length)){ 3127 mc_hash_off = (sym_op->auth.data.offset + 3128 sym_op->auth.data.length); 3129 } 3130 /* for gmac, salt should be updated like in gcm */ 3131 if (unlikely(sess_misc->is_gmac)) { 3132 uint8_t *salt; 3133 salt = fc_params.iv_buf; 3134 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3135 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3136 sess_misc->salt = *(uint32_t *)salt; 3137 } 3138 fc_params.iv_buf = salt + 4; 3139 } 3140 if (likely(sess_misc->mac_len)) { 3141 struct rte_mbuf *m; 3142 3143 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3144 if (!m) 3145 m = m_src; 3146 3147 /* hmac immediately following data is best case */ 3148 if (!ctx->dec_auth && !ctx->auth_enc && 3149 (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3150 mc_hash_off != 3151 (uint8_t *)sym_op->auth.digest.data))) { 3152 flags |= VALID_MAC_BUF; 3153 fc_params.mac_buf.size = 3154 sess_misc->mac_len; 3155 fc_params.mac_buf.vaddr = 3156 sym_op->auth.digest.data; 3157 fc_params.mac_buf.dma_addr = 3158 sym_op->auth.digest.phys_addr; 3159 inplace = 0; 3160 } 3161 } 3162 } 3163 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3164 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3165 3166 if (!ctx->dec_auth && 3167 unlikely(sess_misc->is_null || 3168 sess_misc->cpt_op == CPT_OP_DECODE)) 3169 inplace = 0; 3170 3171 if (likely(!m_dst && inplace)) { 3172 /* Case of single buffer without AAD buf or 3173 * separate mac buf in place and 3174 * not air crypto 3175 */ 3176 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3177 3178 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3179 &fc_params, 3180 &flags))) { 3181 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3182 ret = -EINVAL; 3183 goto err_exit; 3184 } 3185 3186 } else { 3187 /* Out of place processing */ 3188 fc_params.src_iov = (void *)src; 3189 fc_params.dst_iov = (void *)dst; 3190 3191 /* Store SG I/O in the api for reuse */ 3192 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3193 CPT_LOG_DP_ERR("Prepare src iov failed"); 3194 ret = -EINVAL; 3195 goto err_exit; 3196 } 3197 3198 if (unlikely(m_dst != NULL)) { 3199 uint32_t pkt_len; 3200 3201 /* Try to make room as much as src has */ 3202 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3203 3204 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3205 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3206 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3207 CPT_LOG_DP_ERR("Not enough space in " 3208 "m_dst %p, need %u" 3209 " more", 3210 m_dst, pkt_len); 3211 ret = -EINVAL; 3212 goto err_exit; 3213 } 3214 } 3215 3216 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3217 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3218 "m_dst %p", m_dst); 3219 ret = -EINVAL; 3220 goto err_exit; 3221 } 3222 } else { 3223 fc_params.dst_iov = (void *)src; 3224 } 3225 } 3226 3227 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3228 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3229 m_info->lb_mlen, m_info->pool); 3230 else 3231 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3232 m_info->sg_mlen, m_info->pool); 3233 3234 if (unlikely(mdata == NULL)) { 3235 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3236 ret = -ENOMEM; 3237 goto err_exit; 3238 } 3239 3240 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3241 op[0] = (uintptr_t)mdata; 3242 op[1] = (uintptr_t)cop; 3243 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3244 space += 4 * sizeof(uint64_t); 3245 3246 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3247 fc_params.meta_buf.dma_addr += space; 3248 fc_params.meta_buf.size -= space; 3249 3250 /* Finally prepare the instruction */ 3251 if (cpt_op & CPT_OP_ENCODE) 3252 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3253 &fc_params, op); 3254 else 3255 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3256 &fc_params, op); 3257 3258 if (unlikely(*prep_req == NULL)) { 3259 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3260 ret = -EINVAL; 3261 goto free_mdata_and_exit; 3262 } 3263 3264 *mdata_ptr = mdata; 3265 3266 return 0; 3267 3268 free_mdata_and_exit: 3269 free_op_meta(mdata, m_info->pool); 3270 err_exit: 3271 return ret; 3272 } 3273 3274 static __rte_always_inline void 3275 compl_auth_verify(struct rte_crypto_op *op, 3276 uint8_t *gen_mac, 3277 uint64_t mac_len) 3278 { 3279 uint8_t *mac; 3280 struct rte_crypto_sym_op *sym_op = op->sym; 3281 3282 if (sym_op->auth.digest.data) 3283 mac = sym_op->auth.digest.data; 3284 else 3285 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3286 uint8_t *, 3287 sym_op->auth.data.length + 3288 sym_op->auth.data.offset); 3289 if (!mac) { 3290 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3291 return; 3292 } 3293 3294 if (memcmp(mac, gen_mac, mac_len)) 3295 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3296 else 3297 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3298 } 3299 3300 static __rte_always_inline void 3301 find_kasumif9_direction_and_length(uint8_t *src, 3302 uint32_t counter_num_bytes, 3303 uint32_t *addr_length_in_bits, 3304 uint8_t *addr_direction) 3305 { 3306 uint8_t found = 0; 3307 uint32_t pos; 3308 uint8_t last_byte; 3309 while (!found && counter_num_bytes > 0) { 3310 counter_num_bytes--; 3311 if (src[counter_num_bytes] == 0x00) 3312 continue; 3313 pos = rte_bsf32(src[counter_num_bytes]); 3314 if (pos == 7) { 3315 if (likely(counter_num_bytes > 0)) { 3316 last_byte = src[counter_num_bytes - 1]; 3317 *addr_direction = last_byte & 0x1; 3318 *addr_length_in_bits = counter_num_bytes * 8 3319 - 1; 3320 } 3321 } else { 3322 last_byte = src[counter_num_bytes]; 3323 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3324 *addr_length_in_bits = counter_num_bytes * 8 3325 + (8 - (pos + 2)); 3326 } 3327 found = 1; 3328 } 3329 } 3330 3331 /* 3332 * This handles all auth only except AES_GMAC 3333 */ 3334 static __rte_always_inline int 3335 fill_digest_params(struct rte_crypto_op *cop, 3336 struct cpt_sess_misc *sess, 3337 struct cpt_qp_meta_info *m_info, 3338 void **mdata_ptr, 3339 void **prep_req) 3340 { 3341 uint32_t space = 0; 3342 struct rte_crypto_sym_op *sym_op = cop->sym; 3343 void *mdata; 3344 phys_addr_t mphys; 3345 uint64_t *op; 3346 uint32_t auth_range_off; 3347 uint32_t flags = 0; 3348 uint64_t d_offs = 0, d_lens; 3349 struct rte_mbuf *m_src, *m_dst; 3350 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3351 uint16_t mac_len = sess->mac_len; 3352 fc_params_t params; 3353 char src[SRC_IOV_SIZE]; 3354 uint8_t iv_buf[16]; 3355 int ret; 3356 3357 memset(¶ms, 0, sizeof(fc_params_t)); 3358 3359 m_src = sym_op->m_src; 3360 3361 /* For just digest lets force mempool alloc */ 3362 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3363 m_info->pool); 3364 if (mdata == NULL) { 3365 ret = -ENOMEM; 3366 goto err_exit; 3367 } 3368 3369 mphys = params.meta_buf.dma_addr; 3370 3371 op = mdata; 3372 op[0] = (uintptr_t)mdata; 3373 op[1] = (uintptr_t)cop; 3374 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3375 space += 4 * sizeof(uint64_t); 3376 3377 auth_range_off = sym_op->auth.data.offset; 3378 3379 flags = VALID_MAC_BUF; 3380 params.src_iov = (void *)src; 3381 if (unlikely(sess->zsk_flag)) { 3382 /* 3383 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3384 * we will send pass through even for auth only case, 3385 * let MC handle it 3386 */ 3387 d_offs = auth_range_off; 3388 auth_range_off = 0; 3389 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3390 uint8_t *, sess->auth_iv_offset); 3391 if (sess->zsk_flag == K_F9) { 3392 uint32_t length_in_bits, num_bytes; 3393 uint8_t *src, direction = 0; 3394 3395 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3396 uint8_t *), 8); 3397 /* 3398 * This is kasumi f9, take direction from 3399 * source buffer 3400 */ 3401 length_in_bits = cop->sym->auth.data.length; 3402 num_bytes = (length_in_bits >> 3); 3403 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3404 find_kasumif9_direction_and_length(src, 3405 num_bytes, 3406 &length_in_bits, 3407 &direction); 3408 length_in_bits -= 64; 3409 cop->sym->auth.data.offset += 64; 3410 d_offs = cop->sym->auth.data.offset; 3411 auth_range_off = d_offs / 8; 3412 cop->sym->auth.data.length = length_in_bits; 3413 3414 /* Store it at end of auth iv */ 3415 iv_buf[8] = direction; 3416 params.auth_iv_buf = iv_buf; 3417 } 3418 } 3419 3420 d_lens = sym_op->auth.data.length; 3421 3422 params.ctx_buf.vaddr = SESS_PRIV(sess); 3423 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3424 3425 if (auth_op == CPT_OP_AUTH_GENERATE) { 3426 if (sym_op->auth.digest.data) { 3427 /* 3428 * Digest to be generated 3429 * in separate buffer 3430 */ 3431 params.mac_buf.size = 3432 sess->mac_len; 3433 params.mac_buf.vaddr = 3434 sym_op->auth.digest.data; 3435 params.mac_buf.dma_addr = 3436 sym_op->auth.digest.phys_addr; 3437 } else { 3438 uint32_t off = sym_op->auth.data.offset + 3439 sym_op->auth.data.length; 3440 int32_t dlen, space; 3441 3442 m_dst = sym_op->m_dst ? 3443 sym_op->m_dst : sym_op->m_src; 3444 dlen = rte_pktmbuf_pkt_len(m_dst); 3445 3446 space = off + mac_len - dlen; 3447 if (space > 0) 3448 if (!rte_pktmbuf_append(m_dst, space)) { 3449 CPT_LOG_DP_ERR("Failed to extend " 3450 "mbuf by %uB", space); 3451 ret = -EINVAL; 3452 goto free_mdata_and_exit; 3453 } 3454 3455 params.mac_buf.vaddr = 3456 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3457 params.mac_buf.dma_addr = 3458 rte_pktmbuf_iova_offset(m_dst, off); 3459 params.mac_buf.size = mac_len; 3460 } 3461 } else { 3462 /* Need space for storing generated mac */ 3463 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3464 params.mac_buf.dma_addr = mphys + space; 3465 params.mac_buf.size = mac_len; 3466 space += RTE_ALIGN_CEIL(mac_len, 8); 3467 op[2] = (uintptr_t)params.mac_buf.vaddr; 3468 op[3] = mac_len; 3469 } 3470 3471 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3472 params.meta_buf.dma_addr = mphys + space; 3473 params.meta_buf.size -= space; 3474 3475 /* Out of place processing */ 3476 params.src_iov = (void *)src; 3477 3478 /*Store SG I/O in the api for reuse */ 3479 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3480 CPT_LOG_DP_ERR("Prepare src iov failed"); 3481 ret = -EINVAL; 3482 goto free_mdata_and_exit; 3483 } 3484 3485 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3486 if (unlikely(*prep_req == NULL)) { 3487 ret = -EINVAL; 3488 goto free_mdata_and_exit; 3489 } 3490 3491 *mdata_ptr = mdata; 3492 3493 return 0; 3494 3495 free_mdata_and_exit: 3496 free_op_meta(mdata, m_info->pool); 3497 err_exit: 3498 return ret; 3499 } 3500 3501 #endif /*_CPT_UCODE_H_ */ 3502