1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline void 26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 27 { 28 int i, base; 29 30 for (i = 0; i < 4; i++) { 31 base = 4 * i; 32 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 33 (ck[base + 2] << 8) | (ck[base + 3]); 34 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 35 } 36 } 37 38 static __rte_always_inline int 39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) 40 { 41 uint16_t mac_len = auth->digest_length; 42 int ret; 43 44 switch (auth->algo) { 45 case RTE_CRYPTO_AUTH_MD5: 46 case RTE_CRYPTO_AUTH_MD5_HMAC: 47 ret = (mac_len == 16) ? 0 : -1; 48 break; 49 case RTE_CRYPTO_AUTH_SHA1: 50 case RTE_CRYPTO_AUTH_SHA1_HMAC: 51 ret = (mac_len == 20) ? 0 : -1; 52 break; 53 case RTE_CRYPTO_AUTH_SHA224: 54 case RTE_CRYPTO_AUTH_SHA224_HMAC: 55 ret = (mac_len == 28) ? 0 : -1; 56 break; 57 case RTE_CRYPTO_AUTH_SHA256: 58 case RTE_CRYPTO_AUTH_SHA256_HMAC: 59 ret = (mac_len == 32) ? 0 : -1; 60 break; 61 case RTE_CRYPTO_AUTH_SHA384: 62 case RTE_CRYPTO_AUTH_SHA384_HMAC: 63 ret = (mac_len == 48) ? 0 : -1; 64 break; 65 case RTE_CRYPTO_AUTH_SHA512: 66 case RTE_CRYPTO_AUTH_SHA512_HMAC: 67 ret = (mac_len == 64) ? 0 : -1; 68 break; 69 case RTE_CRYPTO_AUTH_NULL: 70 ret = 0; 71 break; 72 default: 73 ret = -1; 74 } 75 76 return ret; 77 } 78 79 static __rte_always_inline void 80 cpt_fc_salt_update(void *ctx, 81 uint8_t *salt) 82 { 83 struct cpt_ctx *cpt_ctx = ctx; 84 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 85 } 86 87 static __rte_always_inline int 88 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 89 { 90 switch (key_len) { 91 case 16: 92 case 24: 93 case 32: 94 return 0; 95 default: 96 return -1; 97 } 98 } 99 100 static __rte_always_inline int 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 102 { 103 int fc_type = 0; 104 switch (type) { 105 case PASSTHROUGH: 106 fc_type = FC_GEN; 107 break; 108 case DES3_CBC: 109 case DES3_ECB: 110 fc_type = FC_GEN; 111 break; 112 case AES_CBC: 113 case AES_ECB: 114 case AES_CFB: 115 case AES_CTR: 116 case AES_GCM: 117 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 118 return -1; 119 fc_type = FC_GEN; 120 break; 121 case CHACHA20: 122 fc_type = FC_GEN; 123 break; 124 case AES_XTS: 125 key_len = key_len / 2; 126 if (unlikely(key_len == 24)) { 127 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 128 return -1; 129 } 130 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 131 return -1; 132 fc_type = FC_GEN; 133 break; 134 case ZUC_EEA3: 135 case SNOW3G_UEA2: 136 if (unlikely(key_len != 16)) 137 return -1; 138 /* No support for AEAD yet */ 139 if (unlikely(ctx->hash_type)) 140 return -1; 141 fc_type = ZUC_SNOW3G; 142 break; 143 case KASUMI_F8_CBC: 144 case KASUMI_F8_ECB: 145 if (unlikely(key_len != 16)) 146 return -1; 147 /* No support for AEAD yet */ 148 if (unlikely(ctx->hash_type)) 149 return -1; 150 fc_type = KASUMI; 151 break; 152 default: 153 return -1; 154 } 155 156 ctx->fc_type = fc_type; 157 return 0; 158 } 159 160 static __rte_always_inline void 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 162 { 163 cpt_ctx->enc_cipher = 0; 164 fctx->enc.enc_cipher = 0; 165 } 166 167 static __rte_always_inline void 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 169 { 170 mc_aes_type_t aes_key_type = 0; 171 switch (key_len) { 172 case 16: 173 aes_key_type = AES_128_BIT; 174 break; 175 case 24: 176 aes_key_type = AES_192_BIT; 177 break; 178 case 32: 179 aes_key_type = AES_256_BIT; 180 break; 181 default: 182 /* This should not happen */ 183 CPT_LOG_DP_ERR("Invalid AES key len"); 184 return; 185 } 186 fctx->enc.aes_key = aes_key_type; 187 } 188 189 static __rte_always_inline void 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 191 uint16_t key_len) 192 { 193 uint32_t keyx[4]; 194 cpt_ctx->snow3g = 1; 195 gen_key_snow3g(key, keyx); 196 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 197 cpt_ctx->zsk_flags = 0; 198 } 199 200 static __rte_always_inline void 201 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 202 uint16_t key_len) 203 { 204 cpt_ctx->snow3g = 0; 205 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 206 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 207 cpt_ctx->zsk_flags = 0; 208 } 209 210 static __rte_always_inline void 211 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 212 uint16_t key_len) 213 { 214 cpt_ctx->k_ecb = 1; 215 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 216 cpt_ctx->zsk_flags = 0; 217 } 218 219 static __rte_always_inline void 220 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 221 uint16_t key_len) 222 { 223 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 224 cpt_ctx->zsk_flags = 0; 225 } 226 227 static __rte_always_inline int 228 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 229 uint16_t key_len, uint8_t *salt) 230 { 231 struct cpt_ctx *cpt_ctx = ctx; 232 mc_fc_context_t *fctx = &cpt_ctx->fctx; 233 int ret; 234 235 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 236 if (unlikely(ret)) 237 return -1; 238 239 if (cpt_ctx->fc_type == FC_GEN) { 240 /* 241 * We need to always say IV is from DPTR as user can 242 * sometimes iverride IV per operation. 243 */ 244 fctx->enc.iv_source = CPT_FROM_DPTR; 245 246 if (cpt_ctx->auth_key_len > 64) 247 return -1; 248 } 249 250 switch (type) { 251 case PASSTHROUGH: 252 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 253 goto success; 254 case DES3_CBC: 255 /* CPT performs DES using 3DES with the 8B DES-key 256 * replicated 2 more times to match the 24B 3DES-key. 257 * Eg. If org. key is "0x0a 0x0b", then new key is 258 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 259 */ 260 if (key_len == 8) { 261 /* Skipping the first 8B as it will be copied 262 * in the regular code flow 263 */ 264 memcpy(fctx->enc.encr_key+key_len, key, key_len); 265 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 266 } 267 break; 268 case DES3_ECB: 269 /* For DES3_ECB IV need to be from CTX. */ 270 fctx->enc.iv_source = CPT_FROM_CTX; 271 break; 272 case AES_CBC: 273 case AES_ECB: 274 case AES_CFB: 275 case AES_CTR: 276 case CHACHA20: 277 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 278 break; 279 case AES_GCM: 280 /* Even though iv source is from dptr, 281 * aes_gcm salt is taken from ctx 282 */ 283 if (salt) { 284 memcpy(fctx->enc.encr_iv, salt, 4); 285 /* Assuming it was just salt update 286 * and nothing else 287 */ 288 if (!key) 289 goto success; 290 } 291 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 292 break; 293 case AES_XTS: 294 key_len = key_len / 2; 295 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 296 297 /* Copy key2 for XTS into ipad */ 298 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 299 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 300 break; 301 case SNOW3G_UEA2: 302 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 303 goto success; 304 case ZUC_EEA3: 305 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 306 goto success; 307 case KASUMI_F8_ECB: 308 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 309 goto success; 310 case KASUMI_F8_CBC: 311 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 312 goto success; 313 default: 314 return -1; 315 } 316 317 /* Only for FC_GEN case */ 318 319 /* For GMAC auth, cipher must be NULL */ 320 if (cpt_ctx->hash_type != GMAC_TYPE) 321 fctx->enc.enc_cipher = type; 322 323 memcpy(fctx->enc.encr_key, key, key_len); 324 325 success: 326 cpt_ctx->enc_cipher = type; 327 328 return 0; 329 } 330 331 static __rte_always_inline uint32_t 332 fill_sg_comp(sg_comp_t *list, 333 uint32_t i, 334 phys_addr_t dma_addr, 335 uint32_t size) 336 { 337 sg_comp_t *to = &list[i>>2]; 338 339 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 340 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 341 i++; 342 return i; 343 } 344 345 static __rte_always_inline uint32_t 346 fill_sg_comp_from_buf(sg_comp_t *list, 347 uint32_t i, 348 buf_ptr_t *from) 349 { 350 sg_comp_t *to = &list[i>>2]; 351 352 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 353 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 354 i++; 355 return i; 356 } 357 358 static __rte_always_inline uint32_t 359 fill_sg_comp_from_buf_min(sg_comp_t *list, 360 uint32_t i, 361 buf_ptr_t *from, 362 uint32_t *psize) 363 { 364 sg_comp_t *to = &list[i >> 2]; 365 uint32_t size = *psize; 366 uint32_t e_len; 367 368 e_len = (size > from->size) ? from->size : size; 369 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 370 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 371 *psize -= e_len; 372 i++; 373 return i; 374 } 375 376 /* 377 * This fills the MC expected SGIO list 378 * from IOV given by user. 379 */ 380 static __rte_always_inline uint32_t 381 fill_sg_comp_from_iov(sg_comp_t *list, 382 uint32_t i, 383 iov_ptr_t *from, uint32_t from_offset, 384 uint32_t *psize, buf_ptr_t *extra_buf, 385 uint32_t extra_offset) 386 { 387 int32_t j; 388 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 389 uint32_t size = *psize; 390 buf_ptr_t *bufs; 391 392 bufs = from->bufs; 393 for (j = 0; (j < from->buf_cnt) && size; j++) { 394 phys_addr_t e_dma_addr; 395 uint32_t e_len; 396 sg_comp_t *to = &list[i >> 2]; 397 398 if (unlikely(from_offset)) { 399 if (from_offset >= bufs[j].size) { 400 from_offset -= bufs[j].size; 401 continue; 402 } 403 e_dma_addr = bufs[j].dma_addr + from_offset; 404 e_len = (size > (bufs[j].size - from_offset)) ? 405 (bufs[j].size - from_offset) : size; 406 from_offset = 0; 407 } else { 408 e_dma_addr = bufs[j].dma_addr; 409 e_len = (size > bufs[j].size) ? 410 bufs[j].size : size; 411 } 412 413 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 414 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 415 416 if (extra_len && (e_len >= extra_offset)) { 417 /* Break the data at given offset */ 418 uint32_t next_len = e_len - extra_offset; 419 phys_addr_t next_dma = e_dma_addr + extra_offset; 420 421 if (!extra_offset) { 422 i--; 423 } else { 424 e_len = extra_offset; 425 size -= e_len; 426 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 427 } 428 429 extra_len = RTE_MIN(extra_len, size); 430 /* Insert extra data ptr */ 431 if (extra_len) { 432 i++; 433 to = &list[i >> 2]; 434 to->u.s.len[i % 4] = 435 rte_cpu_to_be_16(extra_len); 436 to->ptr[i % 4] = 437 rte_cpu_to_be_64(extra_buf->dma_addr); 438 size -= extra_len; 439 } 440 441 next_len = RTE_MIN(next_len, size); 442 /* insert the rest of the data */ 443 if (next_len) { 444 i++; 445 to = &list[i >> 2]; 446 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 447 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 448 size -= next_len; 449 } 450 extra_len = 0; 451 452 } else { 453 size -= e_len; 454 } 455 if (extra_offset) 456 extra_offset -= size; 457 i++; 458 } 459 460 *psize = size; 461 return (uint32_t)i; 462 } 463 464 static __rte_always_inline void 465 cpt_digest_gen_prep(uint32_t flags, 466 uint64_t d_lens, 467 digest_params_t *params, 468 void *op, 469 void **prep_req) 470 { 471 struct cpt_request_info *req; 472 uint32_t size, i; 473 uint16_t data_len, mac_len, key_len; 474 auth_type_t hash_type; 475 buf_ptr_t *meta_p; 476 struct cpt_ctx *ctx; 477 sg_comp_t *gather_comp; 478 sg_comp_t *scatter_comp; 479 uint8_t *in_buffer; 480 uint32_t g_size_bytes, s_size_bytes; 481 uint64_t dptr_dma, rptr_dma; 482 vq_cmd_word0_t vq_cmd_w0; 483 vq_cmd_word3_t vq_cmd_w3; 484 void *c_vaddr, *m_vaddr; 485 uint64_t c_dma, m_dma; 486 opcode_info_t opcode; 487 488 ctx = params->ctx_buf.vaddr; 489 meta_p = ¶ms->meta_buf; 490 491 m_vaddr = meta_p->vaddr; 492 m_dma = meta_p->dma_addr; 493 494 /* 495 * Save initial space that followed app data for completion code & 496 * alternate completion code to fall in same cache line as app data 497 */ 498 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 499 m_dma += COMPLETION_CODE_SIZE; 500 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 501 (uint8_t *)m_vaddr; 502 c_vaddr = (uint8_t *)m_vaddr + size; 503 c_dma = m_dma + size; 504 size += sizeof(cpt_res_s_t); 505 506 m_vaddr = (uint8_t *)m_vaddr + size; 507 m_dma += size; 508 509 req = m_vaddr; 510 511 size = sizeof(struct cpt_request_info); 512 m_vaddr = (uint8_t *)m_vaddr + size; 513 m_dma += size; 514 515 hash_type = ctx->hash_type; 516 mac_len = ctx->mac_len; 517 key_len = ctx->auth_key_len; 518 data_len = AUTH_DLEN(d_lens); 519 520 /*GP op header */ 521 vq_cmd_w0.u64 = 0; 522 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 523 if (ctx->hmac) { 524 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 525 vq_cmd_w0.s.param1 = key_len; 526 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 527 } else { 528 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 529 vq_cmd_w0.s.param1 = 0; 530 vq_cmd_w0.s.dlen = data_len; 531 } 532 533 opcode.s.minor = 0; 534 535 /* Null auth only case enters the if */ 536 if (unlikely(!hash_type && !ctx->enc_cipher)) { 537 opcode.s.major = CPT_MAJOR_OP_MISC; 538 /* Minor op is passthrough */ 539 opcode.s.minor = 0x03; 540 /* Send out completion code only */ 541 vq_cmd_w0.s.param2 = 0x1; 542 } 543 544 vq_cmd_w0.s.opcode = opcode.flags; 545 546 /* DPTR has SG list */ 547 in_buffer = m_vaddr; 548 dptr_dma = m_dma; 549 550 ((uint16_t *)in_buffer)[0] = 0; 551 ((uint16_t *)in_buffer)[1] = 0; 552 553 /* TODO Add error check if space will be sufficient */ 554 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 555 556 /* 557 * Input gather list 558 */ 559 560 i = 0; 561 562 if (ctx->hmac) { 563 uint64_t k_dma = params->ctx_buf.dma_addr + 564 offsetof(struct cpt_ctx, auth_key); 565 /* Key */ 566 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 567 } 568 569 /* input data */ 570 size = data_len; 571 if (size) { 572 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 573 0, &size, NULL, 0); 574 if (unlikely(size)) { 575 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 576 " by %dB", size); 577 return; 578 } 579 } else { 580 /* 581 * Looks like we need to support zero data 582 * gather ptr in case of hash & hmac 583 */ 584 i++; 585 } 586 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 587 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 588 589 /* 590 * Output Gather list 591 */ 592 593 i = 0; 594 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 595 596 if (flags & VALID_MAC_BUF) { 597 if (unlikely(params->mac_buf.size < mac_len)) { 598 CPT_LOG_DP_ERR("Insufficient MAC size"); 599 return; 600 } 601 602 size = mac_len; 603 i = fill_sg_comp_from_buf_min(scatter_comp, i, 604 ¶ms->mac_buf, &size); 605 } else { 606 size = mac_len; 607 i = fill_sg_comp_from_iov(scatter_comp, i, 608 params->src_iov, data_len, 609 &size, NULL, 0); 610 if (unlikely(size)) { 611 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 612 " %dB", size); 613 return; 614 } 615 } 616 617 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 618 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 619 620 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 621 622 /* This is DPTR len incase of SG mode */ 623 vq_cmd_w0.s.dlen = size; 624 625 m_vaddr = (uint8_t *)m_vaddr + size; 626 m_dma += size; 627 628 /* cpt alternate completion address saved earlier */ 629 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 630 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 631 rptr_dma = c_dma - 8; 632 633 req->ist.ei1 = dptr_dma; 634 req->ist.ei2 = rptr_dma; 635 636 /* vq command w3 */ 637 vq_cmd_w3.u64 = 0; 638 639 /* 16 byte aligned cpt res address */ 640 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 641 *req->completion_addr = COMPLETION_CODE_INIT; 642 req->comp_baddr = c_dma; 643 644 /* Fill microcode part of instruction */ 645 req->ist.ei0 = vq_cmd_w0.u64; 646 req->ist.ei3 = vq_cmd_w3.u64; 647 648 req->op = op; 649 650 *prep_req = req; 651 return; 652 } 653 654 static __rte_always_inline void 655 cpt_enc_hmac_prep(uint32_t flags, 656 uint64_t d_offs, 657 uint64_t d_lens, 658 fc_params_t *fc_params, 659 void *op, 660 void **prep_req) 661 { 662 uint32_t iv_offset = 0; 663 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 664 struct cpt_ctx *cpt_ctx; 665 uint32_t cipher_type, hash_type; 666 uint32_t mac_len, size; 667 uint8_t iv_len = 16; 668 struct cpt_request_info *req; 669 buf_ptr_t *meta_p, *aad_buf = NULL; 670 uint32_t encr_offset, auth_offset; 671 uint32_t encr_data_len, auth_data_len, aad_len = 0; 672 uint32_t passthrough_len = 0; 673 void *m_vaddr, *offset_vaddr; 674 uint64_t m_dma, offset_dma, ctx_dma; 675 vq_cmd_word0_t vq_cmd_w0; 676 vq_cmd_word3_t vq_cmd_w3; 677 void *c_vaddr; 678 uint64_t c_dma; 679 opcode_info_t opcode; 680 681 meta_p = &fc_params->meta_buf; 682 m_vaddr = meta_p->vaddr; 683 m_dma = meta_p->dma_addr; 684 685 encr_offset = ENCR_OFFSET(d_offs); 686 auth_offset = AUTH_OFFSET(d_offs); 687 encr_data_len = ENCR_DLEN(d_lens); 688 auth_data_len = AUTH_DLEN(d_lens); 689 if (unlikely(flags & VALID_AAD_BUF)) { 690 /* 691 * We dont support both aad 692 * and auth data separately 693 */ 694 auth_data_len = 0; 695 auth_offset = 0; 696 aad_len = fc_params->aad_buf.size; 697 aad_buf = &fc_params->aad_buf; 698 } 699 cpt_ctx = fc_params->ctx_buf.vaddr; 700 cipher_type = cpt_ctx->enc_cipher; 701 hash_type = cpt_ctx->hash_type; 702 mac_len = cpt_ctx->mac_len; 703 704 /* 705 * Save initial space that followed app data for completion code & 706 * alternate completion code to fall in same cache line as app data 707 */ 708 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 709 m_dma += COMPLETION_CODE_SIZE; 710 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 711 (uint8_t *)m_vaddr; 712 713 c_vaddr = (uint8_t *)m_vaddr + size; 714 c_dma = m_dma + size; 715 size += sizeof(cpt_res_s_t); 716 717 m_vaddr = (uint8_t *)m_vaddr + size; 718 m_dma += size; 719 720 /* start cpt request info struct at 8 byte boundary */ 721 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 722 (uint8_t *)m_vaddr; 723 724 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 725 726 size += sizeof(struct cpt_request_info); 727 m_vaddr = (uint8_t *)m_vaddr + size; 728 m_dma += size; 729 730 if (unlikely(!(flags & VALID_IV_BUF))) { 731 iv_len = 0; 732 iv_offset = ENCR_IV_OFFSET(d_offs); 733 } 734 735 if (unlikely(flags & VALID_AAD_BUF)) { 736 /* 737 * When AAD is given, data above encr_offset is pass through 738 * Since AAD is given as separate pointer and not as offset, 739 * this is a special case as we need to fragment input data 740 * into passthrough + encr_data and then insert AAD in between. 741 */ 742 if (hash_type != GMAC_TYPE) { 743 passthrough_len = encr_offset; 744 auth_offset = passthrough_len + iv_len; 745 encr_offset = passthrough_len + aad_len + iv_len; 746 auth_data_len = aad_len + encr_data_len; 747 } else { 748 passthrough_len = 16 + aad_len; 749 auth_offset = passthrough_len + iv_len; 750 auth_data_len = aad_len; 751 } 752 } else { 753 encr_offset += iv_len; 754 auth_offset += iv_len; 755 } 756 757 /* Encryption */ 758 opcode.s.major = CPT_MAJOR_OP_FC; 759 opcode.s.minor = 0; 760 761 if (hash_type == GMAC_TYPE) { 762 encr_offset = 0; 763 encr_data_len = 0; 764 } 765 766 auth_dlen = auth_offset + auth_data_len; 767 enc_dlen = encr_data_len + encr_offset; 768 if (unlikely(encr_data_len & 0xf)) { 769 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 770 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 771 else if (likely((cipher_type == AES_CBC) || 772 (cipher_type == AES_ECB))) 773 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 774 } 775 776 if (unlikely(auth_dlen > enc_dlen)) { 777 inputlen = auth_dlen; 778 outputlen = auth_dlen + mac_len; 779 } else { 780 inputlen = enc_dlen; 781 outputlen = enc_dlen + mac_len; 782 } 783 784 /* GP op header */ 785 vq_cmd_w0.u64 = 0; 786 vq_cmd_w0.s.param1 = encr_data_len; 787 vq_cmd_w0.s.param2 = auth_data_len; 788 /* 789 * In 83XX since we have a limitation of 790 * IV & Offset control word not part of instruction 791 * and need to be part of Data Buffer, we check if 792 * head room is there and then only do the Direct mode processing 793 */ 794 if (likely((flags & SINGLE_BUF_INPLACE) && 795 (flags & SINGLE_BUF_HEADTAILROOM))) { 796 void *dm_vaddr = fc_params->bufs[0].vaddr; 797 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 798 /* 799 * This flag indicates that there is 24 bytes head room and 800 * 8 bytes tail room available, so that we get to do 801 * DIRECT MODE with limitation 802 */ 803 804 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 805 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 806 807 /* DPTR */ 808 req->ist.ei1 = offset_dma; 809 /* RPTR should just exclude offset control word */ 810 req->ist.ei2 = dm_dma_addr - iv_len; 811 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 812 + outputlen - iv_len); 813 814 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 815 816 vq_cmd_w0.s.opcode = opcode.flags; 817 818 if (likely(iv_len)) { 819 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 820 + OFF_CTRL_LEN); 821 uint64_t *src = fc_params->iv_buf; 822 dest[0] = src[0]; 823 dest[1] = src[1]; 824 } 825 826 *(uint64_t *)offset_vaddr = 827 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 828 ((uint64_t)iv_offset << 8) | 829 ((uint64_t)auth_offset)); 830 831 } else { 832 uint32_t i, g_size_bytes, s_size_bytes; 833 uint64_t dptr_dma, rptr_dma; 834 sg_comp_t *gather_comp; 835 sg_comp_t *scatter_comp; 836 uint8_t *in_buffer; 837 838 /* This falls under strict SG mode */ 839 offset_vaddr = m_vaddr; 840 offset_dma = m_dma; 841 size = OFF_CTRL_LEN + iv_len; 842 843 m_vaddr = (uint8_t *)m_vaddr + size; 844 m_dma += size; 845 846 opcode.s.major |= CPT_DMA_MODE; 847 848 vq_cmd_w0.s.opcode = opcode.flags; 849 850 if (likely(iv_len)) { 851 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 852 + OFF_CTRL_LEN); 853 uint64_t *src = fc_params->iv_buf; 854 dest[0] = src[0]; 855 dest[1] = src[1]; 856 } 857 858 *(uint64_t *)offset_vaddr = 859 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 860 ((uint64_t)iv_offset << 8) | 861 ((uint64_t)auth_offset)); 862 863 /* DPTR has SG list */ 864 in_buffer = m_vaddr; 865 dptr_dma = m_dma; 866 867 ((uint16_t *)in_buffer)[0] = 0; 868 ((uint16_t *)in_buffer)[1] = 0; 869 870 /* TODO Add error check if space will be sufficient */ 871 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 872 873 /* 874 * Input Gather List 875 */ 876 877 i = 0; 878 879 /* Offset control word that includes iv */ 880 i = fill_sg_comp(gather_comp, i, offset_dma, 881 OFF_CTRL_LEN + iv_len); 882 883 /* Add input data */ 884 size = inputlen - iv_len; 885 if (likely(size)) { 886 uint32_t aad_offset = aad_len ? passthrough_len : 0; 887 888 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 889 i = fill_sg_comp_from_buf_min(gather_comp, i, 890 fc_params->bufs, 891 &size); 892 } else { 893 i = fill_sg_comp_from_iov(gather_comp, i, 894 fc_params->src_iov, 895 0, &size, 896 aad_buf, aad_offset); 897 } 898 899 if (unlikely(size)) { 900 CPT_LOG_DP_ERR("Insufficient buffer space," 901 " size %d needed", size); 902 return; 903 } 904 } 905 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 906 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 907 908 /* 909 * Output Scatter list 910 */ 911 i = 0; 912 scatter_comp = 913 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 914 915 /* Add IV */ 916 if (likely(iv_len)) { 917 i = fill_sg_comp(scatter_comp, i, 918 offset_dma + OFF_CTRL_LEN, 919 iv_len); 920 } 921 922 /* output data or output data + digest*/ 923 if (unlikely(flags & VALID_MAC_BUF)) { 924 size = outputlen - iv_len - mac_len; 925 if (size) { 926 uint32_t aad_offset = 927 aad_len ? passthrough_len : 0; 928 929 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 930 i = fill_sg_comp_from_buf_min( 931 scatter_comp, 932 i, 933 fc_params->bufs, 934 &size); 935 } else { 936 i = fill_sg_comp_from_iov(scatter_comp, 937 i, 938 fc_params->dst_iov, 939 0, 940 &size, 941 aad_buf, 942 aad_offset); 943 } 944 if (unlikely(size)) { 945 CPT_LOG_DP_ERR("Insufficient buffer" 946 " space, size %d needed", 947 size); 948 return; 949 } 950 } 951 /* mac_data */ 952 if (mac_len) { 953 i = fill_sg_comp_from_buf(scatter_comp, i, 954 &fc_params->mac_buf); 955 } 956 } else { 957 /* Output including mac */ 958 size = outputlen - iv_len; 959 if (likely(size)) { 960 uint32_t aad_offset = 961 aad_len ? passthrough_len : 0; 962 963 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 964 i = fill_sg_comp_from_buf_min( 965 scatter_comp, 966 i, 967 fc_params->bufs, 968 &size); 969 } else { 970 i = fill_sg_comp_from_iov(scatter_comp, 971 i, 972 fc_params->dst_iov, 973 0, 974 &size, 975 aad_buf, 976 aad_offset); 977 } 978 if (unlikely(size)) { 979 CPT_LOG_DP_ERR("Insufficient buffer" 980 " space, size %d needed", 981 size); 982 return; 983 } 984 } 985 } 986 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 987 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 988 989 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 990 991 /* This is DPTR len incase of SG mode */ 992 vq_cmd_w0.s.dlen = size; 993 994 m_vaddr = (uint8_t *)m_vaddr + size; 995 m_dma += size; 996 997 /* cpt alternate completion address saved earlier */ 998 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 999 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1000 rptr_dma = c_dma - 8; 1001 1002 req->ist.ei1 = dptr_dma; 1003 req->ist.ei2 = rptr_dma; 1004 } 1005 1006 ctx_dma = fc_params->ctx_buf.dma_addr + 1007 offsetof(struct cpt_ctx, fctx); 1008 /* vq command w3 */ 1009 vq_cmd_w3.u64 = 0; 1010 vq_cmd_w3.s.grp = 0; 1011 vq_cmd_w3.s.cptr = ctx_dma; 1012 1013 /* 16 byte aligned cpt res address */ 1014 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1015 *req->completion_addr = COMPLETION_CODE_INIT; 1016 req->comp_baddr = c_dma; 1017 1018 /* Fill microcode part of instruction */ 1019 req->ist.ei0 = vq_cmd_w0.u64; 1020 req->ist.ei3 = vq_cmd_w3.u64; 1021 1022 req->op = op; 1023 1024 *prep_req = req; 1025 return; 1026 } 1027 1028 static __rte_always_inline void 1029 cpt_dec_hmac_prep(uint32_t flags, 1030 uint64_t d_offs, 1031 uint64_t d_lens, 1032 fc_params_t *fc_params, 1033 void *op, 1034 void **prep_req) 1035 { 1036 uint32_t iv_offset = 0, size; 1037 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1038 struct cpt_ctx *cpt_ctx; 1039 int32_t hash_type, mac_len; 1040 uint8_t iv_len = 16; 1041 struct cpt_request_info *req; 1042 buf_ptr_t *meta_p, *aad_buf = NULL; 1043 uint32_t encr_offset, auth_offset; 1044 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1045 uint32_t passthrough_len = 0; 1046 void *m_vaddr, *offset_vaddr; 1047 uint64_t m_dma, offset_dma, ctx_dma; 1048 opcode_info_t opcode; 1049 vq_cmd_word0_t vq_cmd_w0; 1050 vq_cmd_word3_t vq_cmd_w3; 1051 void *c_vaddr; 1052 uint64_t c_dma; 1053 1054 meta_p = &fc_params->meta_buf; 1055 m_vaddr = meta_p->vaddr; 1056 m_dma = meta_p->dma_addr; 1057 1058 encr_offset = ENCR_OFFSET(d_offs); 1059 auth_offset = AUTH_OFFSET(d_offs); 1060 encr_data_len = ENCR_DLEN(d_lens); 1061 auth_data_len = AUTH_DLEN(d_lens); 1062 1063 if (unlikely(flags & VALID_AAD_BUF)) { 1064 /* 1065 * We dont support both aad 1066 * and auth data separately 1067 */ 1068 auth_data_len = 0; 1069 auth_offset = 0; 1070 aad_len = fc_params->aad_buf.size; 1071 aad_buf = &fc_params->aad_buf; 1072 } 1073 1074 cpt_ctx = fc_params->ctx_buf.vaddr; 1075 hash_type = cpt_ctx->hash_type; 1076 mac_len = cpt_ctx->mac_len; 1077 1078 if (unlikely(!(flags & VALID_IV_BUF))) { 1079 iv_len = 0; 1080 iv_offset = ENCR_IV_OFFSET(d_offs); 1081 } 1082 1083 if (unlikely(flags & VALID_AAD_BUF)) { 1084 /* 1085 * When AAD is given, data above encr_offset is pass through 1086 * Since AAD is given as separate pointer and not as offset, 1087 * this is a special case as we need to fragment input data 1088 * into passthrough + encr_data and then insert AAD in between. 1089 */ 1090 if (hash_type != GMAC_TYPE) { 1091 passthrough_len = encr_offset; 1092 auth_offset = passthrough_len + iv_len; 1093 encr_offset = passthrough_len + aad_len + iv_len; 1094 auth_data_len = aad_len + encr_data_len; 1095 } else { 1096 passthrough_len = 16 + aad_len; 1097 auth_offset = passthrough_len + iv_len; 1098 auth_data_len = aad_len; 1099 } 1100 } else { 1101 encr_offset += iv_len; 1102 auth_offset += iv_len; 1103 } 1104 1105 /* 1106 * Save initial space that followed app data for completion code & 1107 * alternate completion code to fall in same cache line as app data 1108 */ 1109 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1110 m_dma += COMPLETION_CODE_SIZE; 1111 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1112 (uint8_t *)m_vaddr; 1113 c_vaddr = (uint8_t *)m_vaddr + size; 1114 c_dma = m_dma + size; 1115 size += sizeof(cpt_res_s_t); 1116 1117 m_vaddr = (uint8_t *)m_vaddr + size; 1118 m_dma += size; 1119 1120 /* start cpt request info structure at 8 byte alignment */ 1121 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1122 (uint8_t *)m_vaddr; 1123 1124 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1125 1126 size += sizeof(struct cpt_request_info); 1127 m_vaddr = (uint8_t *)m_vaddr + size; 1128 m_dma += size; 1129 1130 /* Decryption */ 1131 opcode.s.major = CPT_MAJOR_OP_FC; 1132 opcode.s.minor = 1; 1133 1134 if (hash_type == GMAC_TYPE) { 1135 encr_offset = 0; 1136 encr_data_len = 0; 1137 } 1138 1139 enc_dlen = encr_offset + encr_data_len; 1140 auth_dlen = auth_offset + auth_data_len; 1141 1142 if (auth_dlen > enc_dlen) { 1143 inputlen = auth_dlen + mac_len; 1144 outputlen = auth_dlen; 1145 } else { 1146 inputlen = enc_dlen + mac_len; 1147 outputlen = enc_dlen; 1148 } 1149 1150 vq_cmd_w0.u64 = 0; 1151 vq_cmd_w0.s.param1 = encr_data_len; 1152 vq_cmd_w0.s.param2 = auth_data_len; 1153 1154 /* 1155 * In 83XX since we have a limitation of 1156 * IV & Offset control word not part of instruction 1157 * and need to be part of Data Buffer, we check if 1158 * head room is there and then only do the Direct mode processing 1159 */ 1160 if (likely((flags & SINGLE_BUF_INPLACE) && 1161 (flags & SINGLE_BUF_HEADTAILROOM))) { 1162 void *dm_vaddr = fc_params->bufs[0].vaddr; 1163 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1164 /* 1165 * This flag indicates that there is 24 bytes head room and 1166 * 8 bytes tail room available, so that we get to do 1167 * DIRECT MODE with limitation 1168 */ 1169 1170 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1171 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1172 req->ist.ei1 = offset_dma; 1173 1174 /* RPTR should just exclude offset control word */ 1175 req->ist.ei2 = dm_dma_addr - iv_len; 1176 1177 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1178 outputlen - iv_len); 1179 /* since this is decryption, 1180 * don't touch the content of 1181 * alternate ccode space as it contains 1182 * hmac. 1183 */ 1184 1185 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1186 1187 vq_cmd_w0.s.opcode = opcode.flags; 1188 1189 if (likely(iv_len)) { 1190 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1191 OFF_CTRL_LEN); 1192 uint64_t *src = fc_params->iv_buf; 1193 dest[0] = src[0]; 1194 dest[1] = src[1]; 1195 } 1196 1197 *(uint64_t *)offset_vaddr = 1198 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1199 ((uint64_t)iv_offset << 8) | 1200 ((uint64_t)auth_offset)); 1201 1202 } else { 1203 uint64_t dptr_dma, rptr_dma; 1204 uint32_t g_size_bytes, s_size_bytes; 1205 sg_comp_t *gather_comp; 1206 sg_comp_t *scatter_comp; 1207 uint8_t *in_buffer; 1208 uint8_t i = 0; 1209 1210 /* This falls under strict SG mode */ 1211 offset_vaddr = m_vaddr; 1212 offset_dma = m_dma; 1213 size = OFF_CTRL_LEN + iv_len; 1214 1215 m_vaddr = (uint8_t *)m_vaddr + size; 1216 m_dma += size; 1217 1218 opcode.s.major |= CPT_DMA_MODE; 1219 1220 vq_cmd_w0.s.opcode = opcode.flags; 1221 1222 if (likely(iv_len)) { 1223 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1224 OFF_CTRL_LEN); 1225 uint64_t *src = fc_params->iv_buf; 1226 dest[0] = src[0]; 1227 dest[1] = src[1]; 1228 } 1229 1230 *(uint64_t *)offset_vaddr = 1231 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1232 ((uint64_t)iv_offset << 8) | 1233 ((uint64_t)auth_offset)); 1234 1235 /* DPTR has SG list */ 1236 in_buffer = m_vaddr; 1237 dptr_dma = m_dma; 1238 1239 ((uint16_t *)in_buffer)[0] = 0; 1240 ((uint16_t *)in_buffer)[1] = 0; 1241 1242 /* TODO Add error check if space will be sufficient */ 1243 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1244 1245 /* 1246 * Input Gather List 1247 */ 1248 i = 0; 1249 1250 /* Offset control word that includes iv */ 1251 i = fill_sg_comp(gather_comp, i, offset_dma, 1252 OFF_CTRL_LEN + iv_len); 1253 1254 /* Add input data */ 1255 if (flags & VALID_MAC_BUF) { 1256 size = inputlen - iv_len - mac_len; 1257 if (size) { 1258 /* input data only */ 1259 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1260 i = fill_sg_comp_from_buf_min( 1261 gather_comp, i, 1262 fc_params->bufs, 1263 &size); 1264 } else { 1265 uint32_t aad_offset = aad_len ? 1266 passthrough_len : 0; 1267 1268 i = fill_sg_comp_from_iov(gather_comp, 1269 i, 1270 fc_params->src_iov, 1271 0, &size, 1272 aad_buf, 1273 aad_offset); 1274 } 1275 if (unlikely(size)) { 1276 CPT_LOG_DP_ERR("Insufficient buffer" 1277 " space, size %d needed", 1278 size); 1279 return; 1280 } 1281 } 1282 1283 /* mac data */ 1284 if (mac_len) { 1285 i = fill_sg_comp_from_buf(gather_comp, i, 1286 &fc_params->mac_buf); 1287 } 1288 } else { 1289 /* input data + mac */ 1290 size = inputlen - iv_len; 1291 if (size) { 1292 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1293 i = fill_sg_comp_from_buf_min( 1294 gather_comp, i, 1295 fc_params->bufs, 1296 &size); 1297 } else { 1298 uint32_t aad_offset = aad_len ? 1299 passthrough_len : 0; 1300 1301 if (unlikely(!fc_params->src_iov)) { 1302 CPT_LOG_DP_ERR("Bad input args"); 1303 return; 1304 } 1305 1306 i = fill_sg_comp_from_iov( 1307 gather_comp, i, 1308 fc_params->src_iov, 1309 0, &size, 1310 aad_buf, 1311 aad_offset); 1312 } 1313 1314 if (unlikely(size)) { 1315 CPT_LOG_DP_ERR("Insufficient buffer" 1316 " space, size %d needed", 1317 size); 1318 return; 1319 } 1320 } 1321 } 1322 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1323 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1324 1325 /* 1326 * Output Scatter List 1327 */ 1328 1329 i = 0; 1330 scatter_comp = 1331 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1332 1333 /* Add iv */ 1334 if (iv_len) { 1335 i = fill_sg_comp(scatter_comp, i, 1336 offset_dma + OFF_CTRL_LEN, 1337 iv_len); 1338 } 1339 1340 /* Add output data */ 1341 size = outputlen - iv_len; 1342 if (size) { 1343 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1344 /* handle single buffer here */ 1345 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1346 fc_params->bufs, 1347 &size); 1348 } else { 1349 uint32_t aad_offset = aad_len ? 1350 passthrough_len : 0; 1351 1352 if (unlikely(!fc_params->dst_iov)) { 1353 CPT_LOG_DP_ERR("Bad input args"); 1354 return; 1355 } 1356 1357 i = fill_sg_comp_from_iov(scatter_comp, i, 1358 fc_params->dst_iov, 0, 1359 &size, aad_buf, 1360 aad_offset); 1361 } 1362 1363 if (unlikely(size)) { 1364 CPT_LOG_DP_ERR("Insufficient buffer space," 1365 " size %d needed", size); 1366 return; 1367 } 1368 } 1369 1370 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1371 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1372 1373 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1374 1375 /* This is DPTR len incase of SG mode */ 1376 vq_cmd_w0.s.dlen = size; 1377 1378 m_vaddr = (uint8_t *)m_vaddr + size; 1379 m_dma += size; 1380 1381 /* cpt alternate completion address saved earlier */ 1382 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1383 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1384 rptr_dma = c_dma - 8; 1385 size += COMPLETION_CODE_SIZE; 1386 1387 req->ist.ei1 = dptr_dma; 1388 req->ist.ei2 = rptr_dma; 1389 } 1390 1391 ctx_dma = fc_params->ctx_buf.dma_addr + 1392 offsetof(struct cpt_ctx, fctx); 1393 /* vq command w3 */ 1394 vq_cmd_w3.u64 = 0; 1395 vq_cmd_w3.s.grp = 0; 1396 vq_cmd_w3.s.cptr = ctx_dma; 1397 1398 /* 16 byte aligned cpt res address */ 1399 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1400 *req->completion_addr = COMPLETION_CODE_INIT; 1401 req->comp_baddr = c_dma; 1402 1403 /* Fill microcode part of instruction */ 1404 req->ist.ei0 = vq_cmd_w0.u64; 1405 req->ist.ei3 = vq_cmd_w3.u64; 1406 1407 req->op = op; 1408 1409 *prep_req = req; 1410 return; 1411 } 1412 1413 static __rte_always_inline void 1414 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1415 uint64_t d_offs, 1416 uint64_t d_lens, 1417 fc_params_t *params, 1418 void *op, 1419 void **prep_req) 1420 { 1421 uint32_t size; 1422 int32_t inputlen, outputlen; 1423 struct cpt_ctx *cpt_ctx; 1424 uint32_t mac_len = 0; 1425 uint8_t snow3g, j; 1426 struct cpt_request_info *req; 1427 buf_ptr_t *buf_p; 1428 uint32_t encr_offset = 0, auth_offset = 0; 1429 uint32_t encr_data_len = 0, auth_data_len = 0; 1430 int flags, iv_len = 16; 1431 void *m_vaddr, *c_vaddr; 1432 uint64_t m_dma, c_dma, offset_ctrl; 1433 uint64_t *offset_vaddr, offset_dma; 1434 uint32_t *iv_s, iv[4]; 1435 vq_cmd_word0_t vq_cmd_w0; 1436 vq_cmd_word3_t vq_cmd_w3; 1437 opcode_info_t opcode; 1438 1439 buf_p = ¶ms->meta_buf; 1440 m_vaddr = buf_p->vaddr; 1441 m_dma = buf_p->dma_addr; 1442 1443 cpt_ctx = params->ctx_buf.vaddr; 1444 flags = cpt_ctx->zsk_flags; 1445 mac_len = cpt_ctx->mac_len; 1446 snow3g = cpt_ctx->snow3g; 1447 1448 /* 1449 * Save initial space that followed app data for completion code & 1450 * alternate completion code to fall in same cache line as app data 1451 */ 1452 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1453 m_dma += COMPLETION_CODE_SIZE; 1454 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1455 (uint8_t *)m_vaddr; 1456 1457 c_vaddr = (uint8_t *)m_vaddr + size; 1458 c_dma = m_dma + size; 1459 size += sizeof(cpt_res_s_t); 1460 1461 m_vaddr = (uint8_t *)m_vaddr + size; 1462 m_dma += size; 1463 1464 /* Reserve memory for cpt request info */ 1465 req = m_vaddr; 1466 1467 size = sizeof(struct cpt_request_info); 1468 m_vaddr = (uint8_t *)m_vaddr + size; 1469 m_dma += size; 1470 1471 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1472 1473 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1474 1475 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1476 (0 << 3) | (flags & 0x7)); 1477 1478 if (flags == 0x1) { 1479 /* 1480 * Microcode expects offsets in bytes 1481 * TODO: Rounding off 1482 */ 1483 auth_data_len = AUTH_DLEN(d_lens); 1484 1485 /* EIA3 or UIA2 */ 1486 auth_offset = AUTH_OFFSET(d_offs); 1487 auth_offset = auth_offset / 8; 1488 1489 /* consider iv len */ 1490 auth_offset += iv_len; 1491 1492 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1493 outputlen = mac_len; 1494 1495 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1496 1497 } else { 1498 /* EEA3 or UEA2 */ 1499 /* 1500 * Microcode expects offsets in bytes 1501 * TODO: Rounding off 1502 */ 1503 encr_data_len = ENCR_DLEN(d_lens); 1504 1505 encr_offset = ENCR_OFFSET(d_offs); 1506 encr_offset = encr_offset / 8; 1507 /* consider iv len */ 1508 encr_offset += iv_len; 1509 1510 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1511 outputlen = inputlen; 1512 1513 /* iv offset is 0 */ 1514 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1515 } 1516 1517 /* IV */ 1518 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1519 params->iv_buf; 1520 1521 if (snow3g) { 1522 /* 1523 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1524 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1525 */ 1526 1527 for (j = 0; j < 4; j++) 1528 iv[j] = iv_s[3 - j]; 1529 } else { 1530 /* ZUC doesn't need a swap */ 1531 for (j = 0; j < 4; j++) 1532 iv[j] = iv_s[j]; 1533 } 1534 1535 /* 1536 * GP op header, lengths are expected in bits. 1537 */ 1538 vq_cmd_w0.u64 = 0; 1539 vq_cmd_w0.s.param1 = encr_data_len; 1540 vq_cmd_w0.s.param2 = auth_data_len; 1541 1542 /* 1543 * In 83XX since we have a limitation of 1544 * IV & Offset control word not part of instruction 1545 * and need to be part of Data Buffer, we check if 1546 * head room is there and then only do the Direct mode processing 1547 */ 1548 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1549 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1550 void *dm_vaddr = params->bufs[0].vaddr; 1551 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1552 /* 1553 * This flag indicates that there is 24 bytes head room and 1554 * 8 bytes tail room available, so that we get to do 1555 * DIRECT MODE with limitation 1556 */ 1557 1558 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1559 OFF_CTRL_LEN - iv_len); 1560 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1561 1562 /* DPTR */ 1563 req->ist.ei1 = offset_dma; 1564 /* RPTR should just exclude offset control word */ 1565 req->ist.ei2 = dm_dma_addr - iv_len; 1566 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1567 + outputlen - iv_len); 1568 1569 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1570 1571 vq_cmd_w0.s.opcode = opcode.flags; 1572 1573 if (likely(iv_len)) { 1574 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1575 + OFF_CTRL_LEN); 1576 memcpy(iv_d, iv, 16); 1577 } 1578 1579 *offset_vaddr = offset_ctrl; 1580 } else { 1581 uint32_t i, g_size_bytes, s_size_bytes; 1582 uint64_t dptr_dma, rptr_dma; 1583 sg_comp_t *gather_comp; 1584 sg_comp_t *scatter_comp; 1585 uint8_t *in_buffer; 1586 uint32_t *iv_d; 1587 1588 /* save space for iv */ 1589 offset_vaddr = m_vaddr; 1590 offset_dma = m_dma; 1591 1592 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1593 m_dma += OFF_CTRL_LEN + iv_len; 1594 1595 opcode.s.major |= CPT_DMA_MODE; 1596 1597 vq_cmd_w0.s.opcode = opcode.flags; 1598 1599 /* DPTR has SG list */ 1600 in_buffer = m_vaddr; 1601 dptr_dma = m_dma; 1602 1603 ((uint16_t *)in_buffer)[0] = 0; 1604 ((uint16_t *)in_buffer)[1] = 0; 1605 1606 /* TODO Add error check if space will be sufficient */ 1607 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1608 1609 /* 1610 * Input Gather List 1611 */ 1612 i = 0; 1613 1614 /* Offset control word followed by iv */ 1615 1616 i = fill_sg_comp(gather_comp, i, offset_dma, 1617 OFF_CTRL_LEN + iv_len); 1618 1619 /* iv offset is 0 */ 1620 *offset_vaddr = offset_ctrl; 1621 1622 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1623 memcpy(iv_d, iv, 16); 1624 1625 /* input data */ 1626 size = inputlen - iv_len; 1627 if (size) { 1628 i = fill_sg_comp_from_iov(gather_comp, i, 1629 params->src_iov, 1630 0, &size, NULL, 0); 1631 if (unlikely(size)) { 1632 CPT_LOG_DP_ERR("Insufficient buffer space," 1633 " size %d needed", size); 1634 return; 1635 } 1636 } 1637 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1638 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1639 1640 /* 1641 * Output Scatter List 1642 */ 1643 1644 i = 0; 1645 scatter_comp = 1646 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1647 1648 if (flags == 0x1) { 1649 /* IV in SLIST only for EEA3 & UEA2 */ 1650 iv_len = 0; 1651 } 1652 1653 if (iv_len) { 1654 i = fill_sg_comp(scatter_comp, i, 1655 offset_dma + OFF_CTRL_LEN, iv_len); 1656 } 1657 1658 /* Add output data */ 1659 if (req_flags & VALID_MAC_BUF) { 1660 size = outputlen - iv_len - mac_len; 1661 if (size) { 1662 i = fill_sg_comp_from_iov(scatter_comp, i, 1663 params->dst_iov, 0, 1664 &size, NULL, 0); 1665 1666 if (unlikely(size)) { 1667 CPT_LOG_DP_ERR("Insufficient buffer space," 1668 " size %d needed", size); 1669 return; 1670 } 1671 } 1672 1673 /* mac data */ 1674 if (mac_len) { 1675 i = fill_sg_comp_from_buf(scatter_comp, i, 1676 ¶ms->mac_buf); 1677 } 1678 } else { 1679 /* Output including mac */ 1680 size = outputlen - iv_len; 1681 if (size) { 1682 i = fill_sg_comp_from_iov(scatter_comp, i, 1683 params->dst_iov, 0, 1684 &size, NULL, 0); 1685 1686 if (unlikely(size)) { 1687 CPT_LOG_DP_ERR("Insufficient buffer space," 1688 " size %d needed", size); 1689 return; 1690 } 1691 } 1692 } 1693 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1694 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1695 1696 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1697 1698 /* This is DPTR len incase of SG mode */ 1699 vq_cmd_w0.s.dlen = size; 1700 1701 m_vaddr = (uint8_t *)m_vaddr + size; 1702 m_dma += size; 1703 1704 /* cpt alternate completion address saved earlier */ 1705 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1706 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1707 rptr_dma = c_dma - 8; 1708 1709 req->ist.ei1 = dptr_dma; 1710 req->ist.ei2 = rptr_dma; 1711 } 1712 1713 /* vq command w3 */ 1714 vq_cmd_w3.u64 = 0; 1715 vq_cmd_w3.s.grp = 0; 1716 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1717 offsetof(struct cpt_ctx, zs_ctx); 1718 1719 /* 16 byte aligned cpt res address */ 1720 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1721 *req->completion_addr = COMPLETION_CODE_INIT; 1722 req->comp_baddr = c_dma; 1723 1724 /* Fill microcode part of instruction */ 1725 req->ist.ei0 = vq_cmd_w0.u64; 1726 req->ist.ei3 = vq_cmd_w3.u64; 1727 1728 req->op = op; 1729 1730 *prep_req = req; 1731 return; 1732 } 1733 1734 static __rte_always_inline void 1735 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1736 uint64_t d_offs, 1737 uint64_t d_lens, 1738 fc_params_t *params, 1739 void *op, 1740 void **prep_req) 1741 { 1742 uint32_t size; 1743 int32_t inputlen = 0, outputlen; 1744 struct cpt_ctx *cpt_ctx; 1745 uint8_t snow3g, iv_len = 16; 1746 struct cpt_request_info *req; 1747 buf_ptr_t *buf_p; 1748 uint32_t encr_offset; 1749 uint32_t encr_data_len; 1750 int flags; 1751 void *m_vaddr, *c_vaddr; 1752 uint64_t m_dma, c_dma; 1753 uint64_t *offset_vaddr, offset_dma; 1754 uint32_t *iv_s, iv[4], j; 1755 vq_cmd_word0_t vq_cmd_w0; 1756 vq_cmd_word3_t vq_cmd_w3; 1757 opcode_info_t opcode; 1758 1759 buf_p = ¶ms->meta_buf; 1760 m_vaddr = buf_p->vaddr; 1761 m_dma = buf_p->dma_addr; 1762 1763 /* 1764 * Microcode expects offsets in bytes 1765 * TODO: Rounding off 1766 */ 1767 encr_offset = ENCR_OFFSET(d_offs) / 8; 1768 encr_data_len = ENCR_DLEN(d_lens); 1769 1770 cpt_ctx = params->ctx_buf.vaddr; 1771 flags = cpt_ctx->zsk_flags; 1772 snow3g = cpt_ctx->snow3g; 1773 /* 1774 * Save initial space that followed app data for completion code & 1775 * alternate completion code to fall in same cache line as app data 1776 */ 1777 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1778 m_dma += COMPLETION_CODE_SIZE; 1779 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1780 (uint8_t *)m_vaddr; 1781 1782 c_vaddr = (uint8_t *)m_vaddr + size; 1783 c_dma = m_dma + size; 1784 size += sizeof(cpt_res_s_t); 1785 1786 m_vaddr = (uint8_t *)m_vaddr + size; 1787 m_dma += size; 1788 1789 /* Reserve memory for cpt request info */ 1790 req = m_vaddr; 1791 1792 size = sizeof(struct cpt_request_info); 1793 m_vaddr = (uint8_t *)m_vaddr + size; 1794 m_dma += size; 1795 1796 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1797 1798 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1799 1800 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1801 (0 << 3) | (flags & 0x7)); 1802 1803 /* consider iv len */ 1804 encr_offset += iv_len; 1805 1806 inputlen = encr_offset + 1807 (RTE_ALIGN(encr_data_len, 8) / 8); 1808 outputlen = inputlen; 1809 1810 /* IV */ 1811 iv_s = params->iv_buf; 1812 if (snow3g) { 1813 /* 1814 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1815 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1816 */ 1817 1818 for (j = 0; j < 4; j++) 1819 iv[j] = iv_s[3 - j]; 1820 } else { 1821 /* ZUC doesn't need a swap */ 1822 for (j = 0; j < 4; j++) 1823 iv[j] = iv_s[j]; 1824 } 1825 1826 /* 1827 * GP op header, lengths are expected in bits. 1828 */ 1829 vq_cmd_w0.u64 = 0; 1830 vq_cmd_w0.s.param1 = encr_data_len; 1831 1832 /* 1833 * In 83XX since we have a limitation of 1834 * IV & Offset control word not part of instruction 1835 * and need to be part of Data Buffer, we check if 1836 * head room is there and then only do the Direct mode processing 1837 */ 1838 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1839 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1840 void *dm_vaddr = params->bufs[0].vaddr; 1841 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1842 /* 1843 * This flag indicates that there is 24 bytes head room and 1844 * 8 bytes tail room available, so that we get to do 1845 * DIRECT MODE with limitation 1846 */ 1847 1848 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1849 OFF_CTRL_LEN - iv_len); 1850 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1851 1852 /* DPTR */ 1853 req->ist.ei1 = offset_dma; 1854 /* RPTR should just exclude offset control word */ 1855 req->ist.ei2 = dm_dma_addr - iv_len; 1856 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1857 + outputlen - iv_len); 1858 1859 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1860 1861 vq_cmd_w0.s.opcode = opcode.flags; 1862 1863 if (likely(iv_len)) { 1864 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1865 + OFF_CTRL_LEN); 1866 memcpy(iv_d, iv, 16); 1867 } 1868 1869 /* iv offset is 0 */ 1870 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1871 } else { 1872 uint32_t i, g_size_bytes, s_size_bytes; 1873 uint64_t dptr_dma, rptr_dma; 1874 sg_comp_t *gather_comp; 1875 sg_comp_t *scatter_comp; 1876 uint8_t *in_buffer; 1877 uint32_t *iv_d; 1878 1879 /* save space for offset and iv... */ 1880 offset_vaddr = m_vaddr; 1881 offset_dma = m_dma; 1882 1883 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1884 m_dma += OFF_CTRL_LEN + iv_len; 1885 1886 opcode.s.major |= CPT_DMA_MODE; 1887 1888 vq_cmd_w0.s.opcode = opcode.flags; 1889 1890 /* DPTR has SG list */ 1891 in_buffer = m_vaddr; 1892 dptr_dma = m_dma; 1893 1894 ((uint16_t *)in_buffer)[0] = 0; 1895 ((uint16_t *)in_buffer)[1] = 0; 1896 1897 /* TODO Add error check if space will be sufficient */ 1898 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1899 1900 /* 1901 * Input Gather List 1902 */ 1903 i = 0; 1904 1905 /* Offset control word */ 1906 1907 /* iv offset is 0 */ 1908 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1909 1910 i = fill_sg_comp(gather_comp, i, offset_dma, 1911 OFF_CTRL_LEN + iv_len); 1912 1913 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1914 memcpy(iv_d, iv, 16); 1915 1916 /* Add input data */ 1917 size = inputlen - iv_len; 1918 if (size) { 1919 i = fill_sg_comp_from_iov(gather_comp, i, 1920 params->src_iov, 1921 0, &size, NULL, 0); 1922 if (unlikely(size)) { 1923 CPT_LOG_DP_ERR("Insufficient buffer space," 1924 " size %d needed", size); 1925 return; 1926 } 1927 } 1928 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1929 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1930 1931 /* 1932 * Output Scatter List 1933 */ 1934 1935 i = 0; 1936 scatter_comp = 1937 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1938 1939 /* IV */ 1940 i = fill_sg_comp(scatter_comp, i, 1941 offset_dma + OFF_CTRL_LEN, 1942 iv_len); 1943 1944 /* Add output data */ 1945 size = outputlen - iv_len; 1946 if (size) { 1947 i = fill_sg_comp_from_iov(scatter_comp, i, 1948 params->dst_iov, 0, 1949 &size, NULL, 0); 1950 1951 if (unlikely(size)) { 1952 CPT_LOG_DP_ERR("Insufficient buffer space," 1953 " size %d needed", size); 1954 return; 1955 } 1956 } 1957 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1958 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1959 1960 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1961 1962 /* This is DPTR len incase of SG mode */ 1963 vq_cmd_w0.s.dlen = size; 1964 1965 m_vaddr = (uint8_t *)m_vaddr + size; 1966 m_dma += size; 1967 1968 /* cpt alternate completion address saved earlier */ 1969 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1970 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1971 rptr_dma = c_dma - 8; 1972 1973 req->ist.ei1 = dptr_dma; 1974 req->ist.ei2 = rptr_dma; 1975 } 1976 1977 /* vq command w3 */ 1978 vq_cmd_w3.u64 = 0; 1979 vq_cmd_w3.s.grp = 0; 1980 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1981 offsetof(struct cpt_ctx, zs_ctx); 1982 1983 /* 16 byte aligned cpt res address */ 1984 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1985 *req->completion_addr = COMPLETION_CODE_INIT; 1986 req->comp_baddr = c_dma; 1987 1988 /* Fill microcode part of instruction */ 1989 req->ist.ei0 = vq_cmd_w0.u64; 1990 req->ist.ei3 = vq_cmd_w3.u64; 1991 1992 req->op = op; 1993 1994 *prep_req = req; 1995 return; 1996 } 1997 1998 static __rte_always_inline void 1999 cpt_kasumi_enc_prep(uint32_t req_flags, 2000 uint64_t d_offs, 2001 uint64_t d_lens, 2002 fc_params_t *params, 2003 void *op, 2004 void **prep_req) 2005 { 2006 uint32_t size; 2007 int32_t inputlen = 0, outputlen = 0; 2008 struct cpt_ctx *cpt_ctx; 2009 uint32_t mac_len = 0; 2010 uint8_t i = 0; 2011 struct cpt_request_info *req; 2012 buf_ptr_t *buf_p; 2013 uint32_t encr_offset, auth_offset; 2014 uint32_t encr_data_len, auth_data_len; 2015 int flags; 2016 uint8_t *iv_s, *iv_d, iv_len = 8; 2017 uint8_t dir = 0; 2018 void *m_vaddr, *c_vaddr; 2019 uint64_t m_dma, c_dma; 2020 uint64_t *offset_vaddr, offset_dma; 2021 vq_cmd_word0_t vq_cmd_w0; 2022 vq_cmd_word3_t vq_cmd_w3; 2023 opcode_info_t opcode; 2024 uint8_t *in_buffer; 2025 uint32_t g_size_bytes, s_size_bytes; 2026 uint64_t dptr_dma, rptr_dma; 2027 sg_comp_t *gather_comp; 2028 sg_comp_t *scatter_comp; 2029 2030 buf_p = ¶ms->meta_buf; 2031 m_vaddr = buf_p->vaddr; 2032 m_dma = buf_p->dma_addr; 2033 2034 encr_offset = ENCR_OFFSET(d_offs) / 8; 2035 auth_offset = AUTH_OFFSET(d_offs) / 8; 2036 encr_data_len = ENCR_DLEN(d_lens); 2037 auth_data_len = AUTH_DLEN(d_lens); 2038 2039 cpt_ctx = params->ctx_buf.vaddr; 2040 flags = cpt_ctx->zsk_flags; 2041 mac_len = cpt_ctx->mac_len; 2042 2043 if (flags == 0x0) 2044 iv_s = params->iv_buf; 2045 else 2046 iv_s = params->auth_iv_buf; 2047 2048 dir = iv_s[8] & 0x1; 2049 2050 /* 2051 * Save initial space that followed app data for completion code & 2052 * alternate completion code to fall in same cache line as app data 2053 */ 2054 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2055 m_dma += COMPLETION_CODE_SIZE; 2056 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2057 (uint8_t *)m_vaddr; 2058 2059 c_vaddr = (uint8_t *)m_vaddr + size; 2060 c_dma = m_dma + size; 2061 size += sizeof(cpt_res_s_t); 2062 2063 m_vaddr = (uint8_t *)m_vaddr + size; 2064 m_dma += size; 2065 2066 /* Reserve memory for cpt request info */ 2067 req = m_vaddr; 2068 2069 size = sizeof(struct cpt_request_info); 2070 m_vaddr = (uint8_t *)m_vaddr + size; 2071 m_dma += size; 2072 2073 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2074 2075 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2076 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2077 (dir << 4) | (0 << 3) | (flags & 0x7)); 2078 2079 /* 2080 * GP op header, lengths are expected in bits. 2081 */ 2082 vq_cmd_w0.u64 = 0; 2083 vq_cmd_w0.s.param1 = encr_data_len; 2084 vq_cmd_w0.s.param2 = auth_data_len; 2085 vq_cmd_w0.s.opcode = opcode.flags; 2086 2087 /* consider iv len */ 2088 if (flags == 0x0) { 2089 encr_offset += iv_len; 2090 auth_offset += iv_len; 2091 } 2092 2093 /* save space for offset ctrl and iv */ 2094 offset_vaddr = m_vaddr; 2095 offset_dma = m_dma; 2096 2097 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2098 m_dma += OFF_CTRL_LEN + iv_len; 2099 2100 /* DPTR has SG list */ 2101 in_buffer = m_vaddr; 2102 dptr_dma = m_dma; 2103 2104 ((uint16_t *)in_buffer)[0] = 0; 2105 ((uint16_t *)in_buffer)[1] = 0; 2106 2107 /* TODO Add error check if space will be sufficient */ 2108 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2109 2110 /* 2111 * Input Gather List 2112 */ 2113 i = 0; 2114 2115 /* Offset control word followed by iv */ 2116 2117 if (flags == 0x0) { 2118 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2119 outputlen = inputlen; 2120 /* iv offset is 0 */ 2121 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2122 } else { 2123 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2124 outputlen = mac_len; 2125 /* iv offset is 0 */ 2126 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2127 } 2128 2129 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2130 2131 /* IV */ 2132 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2133 memcpy(iv_d, iv_s, iv_len); 2134 2135 /* input data */ 2136 size = inputlen - iv_len; 2137 if (size) { 2138 i = fill_sg_comp_from_iov(gather_comp, i, 2139 params->src_iov, 0, 2140 &size, NULL, 0); 2141 2142 if (unlikely(size)) { 2143 CPT_LOG_DP_ERR("Insufficient buffer space," 2144 " size %d needed", size); 2145 return; 2146 } 2147 } 2148 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2149 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2150 2151 /* 2152 * Output Scatter List 2153 */ 2154 2155 i = 0; 2156 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2157 2158 if (flags == 0x1) { 2159 /* IV in SLIST only for F8 */ 2160 iv_len = 0; 2161 } 2162 2163 /* IV */ 2164 if (iv_len) { 2165 i = fill_sg_comp(scatter_comp, i, 2166 offset_dma + OFF_CTRL_LEN, 2167 iv_len); 2168 } 2169 2170 /* Add output data */ 2171 if (req_flags & VALID_MAC_BUF) { 2172 size = outputlen - iv_len - mac_len; 2173 if (size) { 2174 i = fill_sg_comp_from_iov(scatter_comp, i, 2175 params->dst_iov, 0, 2176 &size, NULL, 0); 2177 2178 if (unlikely(size)) { 2179 CPT_LOG_DP_ERR("Insufficient buffer space," 2180 " size %d needed", size); 2181 return; 2182 } 2183 } 2184 2185 /* mac data */ 2186 if (mac_len) { 2187 i = fill_sg_comp_from_buf(scatter_comp, i, 2188 ¶ms->mac_buf); 2189 } 2190 } else { 2191 /* Output including mac */ 2192 size = outputlen - iv_len; 2193 if (size) { 2194 i = fill_sg_comp_from_iov(scatter_comp, i, 2195 params->dst_iov, 0, 2196 &size, NULL, 0); 2197 2198 if (unlikely(size)) { 2199 CPT_LOG_DP_ERR("Insufficient buffer space," 2200 " size %d needed", size); 2201 return; 2202 } 2203 } 2204 } 2205 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2206 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2207 2208 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2209 2210 /* This is DPTR len incase of SG mode */ 2211 vq_cmd_w0.s.dlen = size; 2212 2213 m_vaddr = (uint8_t *)m_vaddr + size; 2214 m_dma += size; 2215 2216 /* cpt alternate completion address saved earlier */ 2217 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2218 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2219 rptr_dma = c_dma - 8; 2220 2221 req->ist.ei1 = dptr_dma; 2222 req->ist.ei2 = rptr_dma; 2223 2224 /* vq command w3 */ 2225 vq_cmd_w3.u64 = 0; 2226 vq_cmd_w3.s.grp = 0; 2227 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2228 offsetof(struct cpt_ctx, k_ctx); 2229 2230 /* 16 byte aligned cpt res address */ 2231 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2232 *req->completion_addr = COMPLETION_CODE_INIT; 2233 req->comp_baddr = c_dma; 2234 2235 /* Fill microcode part of instruction */ 2236 req->ist.ei0 = vq_cmd_w0.u64; 2237 req->ist.ei3 = vq_cmd_w3.u64; 2238 2239 req->op = op; 2240 2241 *prep_req = req; 2242 return; 2243 } 2244 2245 static __rte_always_inline void 2246 cpt_kasumi_dec_prep(uint64_t d_offs, 2247 uint64_t d_lens, 2248 fc_params_t *params, 2249 void *op, 2250 void **prep_req) 2251 { 2252 uint32_t size; 2253 int32_t inputlen = 0, outputlen; 2254 struct cpt_ctx *cpt_ctx; 2255 uint8_t i = 0, iv_len = 8; 2256 struct cpt_request_info *req; 2257 buf_ptr_t *buf_p; 2258 uint32_t encr_offset; 2259 uint32_t encr_data_len; 2260 int flags; 2261 uint8_t dir = 0; 2262 void *m_vaddr, *c_vaddr; 2263 uint64_t m_dma, c_dma; 2264 uint64_t *offset_vaddr, offset_dma; 2265 vq_cmd_word0_t vq_cmd_w0; 2266 vq_cmd_word3_t vq_cmd_w3; 2267 opcode_info_t opcode; 2268 uint8_t *in_buffer; 2269 uint32_t g_size_bytes, s_size_bytes; 2270 uint64_t dptr_dma, rptr_dma; 2271 sg_comp_t *gather_comp; 2272 sg_comp_t *scatter_comp; 2273 2274 buf_p = ¶ms->meta_buf; 2275 m_vaddr = buf_p->vaddr; 2276 m_dma = buf_p->dma_addr; 2277 2278 encr_offset = ENCR_OFFSET(d_offs) / 8; 2279 encr_data_len = ENCR_DLEN(d_lens); 2280 2281 cpt_ctx = params->ctx_buf.vaddr; 2282 flags = cpt_ctx->zsk_flags; 2283 /* 2284 * Save initial space that followed app data for completion code & 2285 * alternate completion code to fall in same cache line as app data 2286 */ 2287 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2288 m_dma += COMPLETION_CODE_SIZE; 2289 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2290 (uint8_t *)m_vaddr; 2291 2292 c_vaddr = (uint8_t *)m_vaddr + size; 2293 c_dma = m_dma + size; 2294 size += sizeof(cpt_res_s_t); 2295 2296 m_vaddr = (uint8_t *)m_vaddr + size; 2297 m_dma += size; 2298 2299 /* Reserve memory for cpt request info */ 2300 req = m_vaddr; 2301 2302 size = sizeof(struct cpt_request_info); 2303 m_vaddr = (uint8_t *)m_vaddr + size; 2304 m_dma += size; 2305 2306 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2307 2308 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2309 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2310 (dir << 4) | (0 << 3) | (flags & 0x7)); 2311 2312 /* 2313 * GP op header, lengths are expected in bits. 2314 */ 2315 vq_cmd_w0.u64 = 0; 2316 vq_cmd_w0.s.param1 = encr_data_len; 2317 vq_cmd_w0.s.opcode = opcode.flags; 2318 2319 /* consider iv len */ 2320 encr_offset += iv_len; 2321 2322 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2323 outputlen = inputlen; 2324 2325 /* save space for offset ctrl & iv */ 2326 offset_vaddr = m_vaddr; 2327 offset_dma = m_dma; 2328 2329 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2330 m_dma += OFF_CTRL_LEN + iv_len; 2331 2332 /* DPTR has SG list */ 2333 in_buffer = m_vaddr; 2334 dptr_dma = m_dma; 2335 2336 ((uint16_t *)in_buffer)[0] = 0; 2337 ((uint16_t *)in_buffer)[1] = 0; 2338 2339 /* TODO Add error check if space will be sufficient */ 2340 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2341 2342 /* 2343 * Input Gather List 2344 */ 2345 i = 0; 2346 2347 /* Offset control word followed by iv */ 2348 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2349 2350 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2351 2352 /* IV */ 2353 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2354 params->iv_buf, iv_len); 2355 2356 /* Add input data */ 2357 size = inputlen - iv_len; 2358 if (size) { 2359 i = fill_sg_comp_from_iov(gather_comp, i, 2360 params->src_iov, 2361 0, &size, NULL, 0); 2362 if (unlikely(size)) { 2363 CPT_LOG_DP_ERR("Insufficient buffer space," 2364 " size %d needed", size); 2365 return; 2366 } 2367 } 2368 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2369 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2370 2371 /* 2372 * Output Scatter List 2373 */ 2374 2375 i = 0; 2376 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2377 2378 /* IV */ 2379 i = fill_sg_comp(scatter_comp, i, 2380 offset_dma + OFF_CTRL_LEN, 2381 iv_len); 2382 2383 /* Add output data */ 2384 size = outputlen - iv_len; 2385 if (size) { 2386 i = fill_sg_comp_from_iov(scatter_comp, i, 2387 params->dst_iov, 0, 2388 &size, NULL, 0); 2389 if (unlikely(size)) { 2390 CPT_LOG_DP_ERR("Insufficient buffer space," 2391 " size %d needed", size); 2392 return; 2393 } 2394 } 2395 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2396 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2397 2398 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2399 2400 /* This is DPTR len incase of SG mode */ 2401 vq_cmd_w0.s.dlen = size; 2402 2403 m_vaddr = (uint8_t *)m_vaddr + size; 2404 m_dma += size; 2405 2406 /* cpt alternate completion address saved earlier */ 2407 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2408 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2409 rptr_dma = c_dma - 8; 2410 2411 req->ist.ei1 = dptr_dma; 2412 req->ist.ei2 = rptr_dma; 2413 2414 /* vq command w3 */ 2415 vq_cmd_w3.u64 = 0; 2416 vq_cmd_w3.s.grp = 0; 2417 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2418 offsetof(struct cpt_ctx, k_ctx); 2419 2420 /* 16 byte aligned cpt res address */ 2421 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2422 *req->completion_addr = COMPLETION_CODE_INIT; 2423 req->comp_baddr = c_dma; 2424 2425 /* Fill microcode part of instruction */ 2426 req->ist.ei0 = vq_cmd_w0.u64; 2427 req->ist.ei3 = vq_cmd_w3.u64; 2428 2429 req->op = op; 2430 2431 *prep_req = req; 2432 return; 2433 } 2434 2435 static __rte_always_inline void * 2436 cpt_fc_dec_hmac_prep(uint32_t flags, 2437 uint64_t d_offs, 2438 uint64_t d_lens, 2439 fc_params_t *fc_params, 2440 void *op) 2441 { 2442 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2443 uint8_t fc_type; 2444 void *prep_req = NULL; 2445 2446 fc_type = ctx->fc_type; 2447 2448 if (likely(fc_type == FC_GEN)) { 2449 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2450 &prep_req); 2451 } else if (fc_type == ZUC_SNOW3G) { 2452 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2453 &prep_req); 2454 } else if (fc_type == KASUMI) { 2455 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2456 } 2457 2458 /* 2459 * For AUTH_ONLY case, 2460 * MC only supports digest generation and verification 2461 * should be done in software by memcmp() 2462 */ 2463 2464 return prep_req; 2465 } 2466 2467 static __rte_always_inline void *__rte_hot 2468 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2469 fc_params_t *fc_params, void *op) 2470 { 2471 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2472 uint8_t fc_type; 2473 void *prep_req = NULL; 2474 2475 fc_type = ctx->fc_type; 2476 2477 /* Common api for rest of the ops */ 2478 if (likely(fc_type == FC_GEN)) { 2479 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2480 &prep_req); 2481 } else if (fc_type == ZUC_SNOW3G) { 2482 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2483 &prep_req); 2484 } else if (fc_type == KASUMI) { 2485 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2486 &prep_req); 2487 } else if (fc_type == HASH_HMAC) { 2488 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2489 } 2490 2491 return prep_req; 2492 } 2493 2494 static __rte_always_inline int 2495 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2496 uint16_t key_len, uint16_t mac_len) 2497 { 2498 struct cpt_ctx *cpt_ctx = ctx; 2499 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2500 2501 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2502 uint32_t keyx[4]; 2503 2504 if (key_len != 16) 2505 return -1; 2506 /* No support for AEAD yet */ 2507 if (cpt_ctx->enc_cipher) 2508 return -1; 2509 /* For ZUC/SNOW3G/Kasumi */ 2510 switch (type) { 2511 case SNOW3G_UIA2: 2512 cpt_ctx->snow3g = 1; 2513 gen_key_snow3g(key, keyx); 2514 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2515 cpt_ctx->fc_type = ZUC_SNOW3G; 2516 cpt_ctx->zsk_flags = 0x1; 2517 break; 2518 case ZUC_EIA3: 2519 cpt_ctx->snow3g = 0; 2520 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2521 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2522 cpt_ctx->fc_type = ZUC_SNOW3G; 2523 cpt_ctx->zsk_flags = 0x1; 2524 break; 2525 case KASUMI_F9_ECB: 2526 /* Kasumi ECB mode */ 2527 cpt_ctx->k_ecb = 1; 2528 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2529 cpt_ctx->fc_type = KASUMI; 2530 cpt_ctx->zsk_flags = 0x1; 2531 break; 2532 case KASUMI_F9_CBC: 2533 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2534 cpt_ctx->fc_type = KASUMI; 2535 cpt_ctx->zsk_flags = 0x1; 2536 break; 2537 default: 2538 return -1; 2539 } 2540 cpt_ctx->mac_len = 4; 2541 cpt_ctx->hash_type = type; 2542 return 0; 2543 } 2544 2545 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2546 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2547 cpt_ctx->fc_type = HASH_HMAC; 2548 } 2549 2550 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2551 return -1; 2552 2553 /* For GMAC auth, cipher must be NULL */ 2554 if (type == GMAC_TYPE) 2555 fctx->enc.enc_cipher = 0; 2556 2557 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2558 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2559 2560 if (key_len) { 2561 cpt_ctx->hmac = 1; 2562 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2563 memcpy(cpt_ctx->auth_key, key, key_len); 2564 cpt_ctx->auth_key_len = key_len; 2565 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2566 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2567 2568 if (key_len <= 64) 2569 memcpy(fctx->hmac.opad, key, key_len); 2570 fctx->enc.auth_input_type = 1; 2571 } 2572 return 0; 2573 } 2574 2575 static __rte_always_inline int 2576 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2577 struct cpt_sess_misc *sess) 2578 { 2579 struct rte_crypto_aead_xform *aead_form; 2580 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2581 auth_type_t auth_type = 0; /* NULL Auth type */ 2582 uint32_t cipher_key_len = 0; 2583 uint8_t aes_gcm = 0; 2584 aead_form = &xform->aead; 2585 void *ctx = SESS_PRIV(sess); 2586 2587 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 2588 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2589 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2590 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) { 2591 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2592 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2593 } else { 2594 CPT_LOG_DP_ERR("Unknown aead operation\n"); 2595 return -1; 2596 } 2597 switch (aead_form->algo) { 2598 case RTE_CRYPTO_AEAD_AES_GCM: 2599 enc_type = AES_GCM; 2600 cipher_key_len = 16; 2601 aes_gcm = 1; 2602 break; 2603 case RTE_CRYPTO_AEAD_AES_CCM: 2604 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2605 aead_form->algo); 2606 return -1; 2607 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: 2608 enc_type = CHACHA20; 2609 auth_type = POLY1305; 2610 cipher_key_len = 32; 2611 sess->chacha_poly = 1; 2612 break; 2613 default: 2614 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2615 aead_form->algo); 2616 return -1; 2617 } 2618 if (aead_form->key.length < cipher_key_len) { 2619 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2620 (unsigned int long)aead_form->key.length); 2621 return -1; 2622 } 2623 sess->zsk_flag = 0; 2624 sess->aes_gcm = aes_gcm; 2625 sess->mac_len = aead_form->digest_length; 2626 sess->iv_offset = aead_form->iv.offset; 2627 sess->iv_length = aead_form->iv.length; 2628 sess->aad_length = aead_form->aad_length; 2629 2630 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2631 aead_form->key.length, NULL))) 2632 return -1; 2633 2634 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2635 aead_form->digest_length))) 2636 return -1; 2637 2638 return 0; 2639 } 2640 2641 static __rte_always_inline int 2642 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2643 struct cpt_sess_misc *sess) 2644 { 2645 struct rte_crypto_cipher_xform *c_form; 2646 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2647 uint32_t cipher_key_len = 0; 2648 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2649 2650 c_form = &xform->cipher; 2651 2652 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2653 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2654 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2655 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2656 else { 2657 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2658 return -1; 2659 } 2660 2661 switch (c_form->algo) { 2662 case RTE_CRYPTO_CIPHER_AES_CBC: 2663 enc_type = AES_CBC; 2664 cipher_key_len = 16; 2665 break; 2666 case RTE_CRYPTO_CIPHER_3DES_CBC: 2667 enc_type = DES3_CBC; 2668 cipher_key_len = 24; 2669 break; 2670 case RTE_CRYPTO_CIPHER_DES_CBC: 2671 /* DES is implemented using 3DES in hardware */ 2672 enc_type = DES3_CBC; 2673 cipher_key_len = 8; 2674 break; 2675 case RTE_CRYPTO_CIPHER_AES_CTR: 2676 enc_type = AES_CTR; 2677 cipher_key_len = 16; 2678 aes_ctr = 1; 2679 break; 2680 case RTE_CRYPTO_CIPHER_NULL: 2681 enc_type = 0; 2682 is_null = 1; 2683 break; 2684 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2685 enc_type = KASUMI_F8_ECB; 2686 cipher_key_len = 16; 2687 zsk_flag = K_F8; 2688 break; 2689 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2690 enc_type = SNOW3G_UEA2; 2691 cipher_key_len = 16; 2692 zsk_flag = ZS_EA; 2693 break; 2694 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2695 enc_type = ZUC_EEA3; 2696 cipher_key_len = 16; 2697 zsk_flag = ZS_EA; 2698 break; 2699 case RTE_CRYPTO_CIPHER_AES_XTS: 2700 enc_type = AES_XTS; 2701 cipher_key_len = 16; 2702 break; 2703 case RTE_CRYPTO_CIPHER_3DES_ECB: 2704 enc_type = DES3_ECB; 2705 cipher_key_len = 24; 2706 break; 2707 case RTE_CRYPTO_CIPHER_AES_ECB: 2708 enc_type = AES_ECB; 2709 cipher_key_len = 16; 2710 break; 2711 case RTE_CRYPTO_CIPHER_3DES_CTR: 2712 case RTE_CRYPTO_CIPHER_AES_F8: 2713 case RTE_CRYPTO_CIPHER_ARC4: 2714 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2715 c_form->algo); 2716 return -1; 2717 default: 2718 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2719 c_form->algo); 2720 return -1; 2721 } 2722 2723 if (c_form->key.length < cipher_key_len) { 2724 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2725 (unsigned long) c_form->key.length); 2726 return -1; 2727 } 2728 2729 sess->zsk_flag = zsk_flag; 2730 sess->aes_gcm = 0; 2731 sess->aes_ctr = aes_ctr; 2732 sess->iv_offset = c_form->iv.offset; 2733 sess->iv_length = c_form->iv.length; 2734 sess->is_null = is_null; 2735 2736 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2737 c_form->key.data, c_form->key.length, NULL))) 2738 return -1; 2739 2740 return 0; 2741 } 2742 2743 static __rte_always_inline int 2744 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2745 struct cpt_sess_misc *sess) 2746 { 2747 struct rte_crypto_auth_xform *a_form; 2748 auth_type_t auth_type = 0; /* NULL Auth type */ 2749 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2750 2751 a_form = &xform->auth; 2752 2753 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2754 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2755 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2756 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2757 else { 2758 CPT_LOG_DP_ERR("Unknown auth operation"); 2759 return -1; 2760 } 2761 2762 switch (a_form->algo) { 2763 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2764 /* Fall through */ 2765 case RTE_CRYPTO_AUTH_SHA1: 2766 auth_type = SHA1_TYPE; 2767 break; 2768 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2769 case RTE_CRYPTO_AUTH_SHA256: 2770 auth_type = SHA2_SHA256; 2771 break; 2772 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2773 case RTE_CRYPTO_AUTH_SHA512: 2774 auth_type = SHA2_SHA512; 2775 break; 2776 case RTE_CRYPTO_AUTH_AES_GMAC: 2777 auth_type = GMAC_TYPE; 2778 aes_gcm = 1; 2779 break; 2780 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2781 case RTE_CRYPTO_AUTH_SHA224: 2782 auth_type = SHA2_SHA224; 2783 break; 2784 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2785 case RTE_CRYPTO_AUTH_SHA384: 2786 auth_type = SHA2_SHA384; 2787 break; 2788 case RTE_CRYPTO_AUTH_MD5_HMAC: 2789 case RTE_CRYPTO_AUTH_MD5: 2790 auth_type = MD5_TYPE; 2791 break; 2792 case RTE_CRYPTO_AUTH_KASUMI_F9: 2793 auth_type = KASUMI_F9_ECB; 2794 /* 2795 * Indicate that direction needs to be taken out 2796 * from end of src 2797 */ 2798 zsk_flag = K_F9; 2799 break; 2800 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2801 auth_type = SNOW3G_UIA2; 2802 zsk_flag = ZS_IA; 2803 break; 2804 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2805 auth_type = ZUC_EIA3; 2806 zsk_flag = ZS_IA; 2807 break; 2808 case RTE_CRYPTO_AUTH_NULL: 2809 auth_type = 0; 2810 is_null = 1; 2811 break; 2812 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2813 case RTE_CRYPTO_AUTH_AES_CMAC: 2814 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2815 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2816 a_form->algo); 2817 return -1; 2818 default: 2819 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2820 a_form->algo); 2821 return -1; 2822 } 2823 2824 sess->zsk_flag = zsk_flag; 2825 sess->aes_gcm = aes_gcm; 2826 sess->mac_len = a_form->digest_length; 2827 sess->is_null = is_null; 2828 if (zsk_flag) { 2829 sess->auth_iv_offset = a_form->iv.offset; 2830 sess->auth_iv_length = a_form->iv.length; 2831 } 2832 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2833 a_form->key.data, a_form->key.length, 2834 a_form->digest_length))) 2835 return -1; 2836 2837 return 0; 2838 } 2839 2840 static __rte_always_inline int 2841 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2842 struct cpt_sess_misc *sess) 2843 { 2844 struct rte_crypto_auth_xform *a_form; 2845 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2846 auth_type_t auth_type = 0; /* NULL Auth type */ 2847 void *ctx = SESS_PRIV(sess); 2848 2849 a_form = &xform->auth; 2850 2851 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2852 sess->cpt_op |= CPT_OP_ENCODE; 2853 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2854 sess->cpt_op |= CPT_OP_DECODE; 2855 else { 2856 CPT_LOG_DP_ERR("Unknown auth operation"); 2857 return -1; 2858 } 2859 2860 switch (a_form->algo) { 2861 case RTE_CRYPTO_AUTH_AES_GMAC: 2862 enc_type = AES_GCM; 2863 auth_type = GMAC_TYPE; 2864 break; 2865 default: 2866 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2867 a_form->algo); 2868 return -1; 2869 } 2870 2871 sess->zsk_flag = 0; 2872 sess->aes_gcm = 0; 2873 sess->is_gmac = 1; 2874 sess->iv_offset = a_form->iv.offset; 2875 sess->iv_length = a_form->iv.length; 2876 sess->mac_len = a_form->digest_length; 2877 2878 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2879 a_form->key.length, NULL))) 2880 return -1; 2881 2882 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2883 a_form->digest_length))) 2884 return -1; 2885 2886 return 0; 2887 } 2888 2889 static __rte_always_inline void * 2890 alloc_op_meta(struct rte_mbuf *m_src, 2891 buf_ptr_t *buf, 2892 int32_t len, 2893 struct rte_mempool *cpt_meta_pool) 2894 { 2895 uint8_t *mdata; 2896 2897 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2898 if (likely(m_src && (m_src->nb_segs == 1))) { 2899 int32_t tailroom; 2900 phys_addr_t mphys; 2901 2902 /* Check if tailroom is sufficient to hold meta data */ 2903 tailroom = rte_pktmbuf_tailroom(m_src); 2904 if (likely(tailroom > len + 8)) { 2905 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2906 mphys = m_src->buf_iova + m_src->buf_len; 2907 mdata -= len; 2908 mphys -= len; 2909 buf->vaddr = mdata; 2910 buf->dma_addr = mphys; 2911 buf->size = len; 2912 /* Indicate that this is a mbuf allocated mdata */ 2913 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2914 return mdata; 2915 } 2916 } 2917 #else 2918 RTE_SET_USED(m_src); 2919 #endif 2920 2921 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2922 return NULL; 2923 2924 buf->vaddr = mdata; 2925 buf->dma_addr = rte_mempool_virt2iova(mdata); 2926 buf->size = len; 2927 2928 return mdata; 2929 } 2930 2931 /** 2932 * cpt_free_metabuf - free metabuf to mempool. 2933 * @param instance: pointer to instance. 2934 * @param objp: pointer to the metabuf. 2935 */ 2936 static __rte_always_inline void 2937 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2938 { 2939 bool nofree = ((uintptr_t)mdata & 1ull); 2940 2941 if (likely(nofree)) 2942 return; 2943 rte_mempool_put(cpt_meta_pool, mdata); 2944 } 2945 2946 static __rte_always_inline uint32_t 2947 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2948 iov_ptr_t *iovec, uint32_t start_offset) 2949 { 2950 uint16_t index = 0; 2951 void *seg_data = NULL; 2952 phys_addr_t seg_phys; 2953 int32_t seg_size = 0; 2954 2955 if (!pkt) { 2956 iovec->buf_cnt = 0; 2957 return 0; 2958 } 2959 2960 if (!start_offset) { 2961 seg_data = rte_pktmbuf_mtod(pkt, void *); 2962 seg_phys = rte_pktmbuf_iova(pkt); 2963 seg_size = pkt->data_len; 2964 } else { 2965 while (start_offset >= pkt->data_len) { 2966 start_offset -= pkt->data_len; 2967 pkt = pkt->next; 2968 } 2969 2970 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2971 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset); 2972 seg_size = pkt->data_len - start_offset; 2973 if (!seg_size) 2974 return 1; 2975 } 2976 2977 /* first seg */ 2978 iovec->bufs[index].vaddr = seg_data; 2979 iovec->bufs[index].dma_addr = seg_phys; 2980 iovec->bufs[index].size = seg_size; 2981 index++; 2982 pkt = pkt->next; 2983 2984 while (unlikely(pkt != NULL)) { 2985 seg_data = rte_pktmbuf_mtod(pkt, void *); 2986 seg_phys = rte_pktmbuf_iova(pkt); 2987 seg_size = pkt->data_len; 2988 if (!seg_size) 2989 break; 2990 2991 iovec->bufs[index].vaddr = seg_data; 2992 iovec->bufs[index].dma_addr = seg_phys; 2993 iovec->bufs[index].size = seg_size; 2994 2995 index++; 2996 2997 pkt = pkt->next; 2998 } 2999 3000 iovec->buf_cnt = index; 3001 return 0; 3002 } 3003 3004 static __rte_always_inline uint32_t 3005 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 3006 fc_params_t *param, 3007 uint32_t *flags) 3008 { 3009 uint16_t index = 0; 3010 void *seg_data = NULL; 3011 phys_addr_t seg_phys; 3012 uint32_t seg_size = 0; 3013 iov_ptr_t *iovec; 3014 3015 seg_data = rte_pktmbuf_mtod(pkt, void *); 3016 seg_phys = rte_pktmbuf_iova(pkt); 3017 seg_size = pkt->data_len; 3018 3019 /* first seg */ 3020 if (likely(!pkt->next)) { 3021 uint32_t headroom, tailroom; 3022 3023 *flags |= SINGLE_BUF_INPLACE; 3024 headroom = rte_pktmbuf_headroom(pkt); 3025 tailroom = rte_pktmbuf_tailroom(pkt); 3026 if (likely((headroom >= 24) && 3027 (tailroom >= 8))) { 3028 /* In 83XX this is prerequivisit for Direct mode */ 3029 *flags |= SINGLE_BUF_HEADTAILROOM; 3030 } 3031 param->bufs[0].vaddr = seg_data; 3032 param->bufs[0].dma_addr = seg_phys; 3033 param->bufs[0].size = seg_size; 3034 return 0; 3035 } 3036 iovec = param->src_iov; 3037 iovec->bufs[index].vaddr = seg_data; 3038 iovec->bufs[index].dma_addr = seg_phys; 3039 iovec->bufs[index].size = seg_size; 3040 index++; 3041 pkt = pkt->next; 3042 3043 while (unlikely(pkt != NULL)) { 3044 seg_data = rte_pktmbuf_mtod(pkt, void *); 3045 seg_phys = rte_pktmbuf_iova(pkt); 3046 seg_size = pkt->data_len; 3047 3048 if (!seg_size) 3049 break; 3050 3051 iovec->bufs[index].vaddr = seg_data; 3052 iovec->bufs[index].dma_addr = seg_phys; 3053 iovec->bufs[index].size = seg_size; 3054 3055 index++; 3056 3057 pkt = pkt->next; 3058 } 3059 3060 iovec->buf_cnt = index; 3061 return 0; 3062 } 3063 3064 static __rte_always_inline int 3065 fill_fc_params(struct rte_crypto_op *cop, 3066 struct cpt_sess_misc *sess_misc, 3067 struct cpt_qp_meta_info *m_info, 3068 void **mdata_ptr, 3069 void **prep_req) 3070 { 3071 uint32_t space = 0; 3072 struct rte_crypto_sym_op *sym_op = cop->sym; 3073 void *mdata = NULL; 3074 uintptr_t *op; 3075 uint32_t mc_hash_off; 3076 uint32_t flags = 0; 3077 uint64_t d_offs, d_lens; 3078 struct rte_mbuf *m_src, *m_dst; 3079 uint8_t cpt_op = sess_misc->cpt_op; 3080 #ifdef CPT_ALWAYS_USE_SG_MODE 3081 uint8_t inplace = 0; 3082 #else 3083 uint8_t inplace = 1; 3084 #endif 3085 fc_params_t fc_params; 3086 char src[SRC_IOV_SIZE]; 3087 char dst[SRC_IOV_SIZE]; 3088 uint32_t iv_buf[4]; 3089 int ret; 3090 3091 if (likely(sess_misc->iv_length)) { 3092 flags |= VALID_IV_BUF; 3093 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3094 uint8_t *, sess_misc->iv_offset); 3095 if (sess_misc->aes_ctr && 3096 unlikely(sess_misc->iv_length != 16)) { 3097 memcpy((uint8_t *)iv_buf, 3098 rte_crypto_op_ctod_offset(cop, 3099 uint8_t *, sess_misc->iv_offset), 12); 3100 iv_buf[3] = rte_cpu_to_be_32(0x1); 3101 fc_params.iv_buf = iv_buf; 3102 } 3103 } 3104 3105 if (sess_misc->zsk_flag) { 3106 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3107 uint8_t *, 3108 sess_misc->auth_iv_offset); 3109 if (sess_misc->zsk_flag != ZS_EA) 3110 inplace = 0; 3111 } 3112 m_src = sym_op->m_src; 3113 m_dst = sym_op->m_dst; 3114 3115 if (sess_misc->aes_gcm || sess_misc->chacha_poly) { 3116 uint8_t *salt; 3117 uint8_t *aad_data; 3118 uint16_t aad_len; 3119 3120 d_offs = sym_op->aead.data.offset; 3121 d_lens = sym_op->aead.data.length; 3122 mc_hash_off = sym_op->aead.data.offset + 3123 sym_op->aead.data.length; 3124 3125 aad_data = sym_op->aead.aad.data; 3126 aad_len = sess_misc->aad_length; 3127 if (likely((aad_data + aad_len) == 3128 rte_pktmbuf_mtod_offset(m_src, 3129 uint8_t *, 3130 sym_op->aead.data.offset))) { 3131 d_offs = (d_offs - aad_len) | (d_offs << 16); 3132 d_lens = (d_lens + aad_len) | (d_lens << 32); 3133 } else { 3134 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3135 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3136 fc_params.aad_buf.size = aad_len; 3137 flags |= VALID_AAD_BUF; 3138 inplace = 0; 3139 d_offs = d_offs << 16; 3140 d_lens = d_lens << 32; 3141 } 3142 3143 salt = fc_params.iv_buf; 3144 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3145 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3146 sess_misc->salt = *(uint32_t *)salt; 3147 } 3148 fc_params.iv_buf = salt + 4; 3149 if (likely(sess_misc->mac_len)) { 3150 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3151 m_src; 3152 3153 if (!m) 3154 m = m_src; 3155 3156 /* hmac immediately following data is best case */ 3157 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3158 mc_hash_off != 3159 (uint8_t *)sym_op->aead.digest.data)) { 3160 flags |= VALID_MAC_BUF; 3161 fc_params.mac_buf.size = sess_misc->mac_len; 3162 fc_params.mac_buf.vaddr = 3163 sym_op->aead.digest.data; 3164 fc_params.mac_buf.dma_addr = 3165 sym_op->aead.digest.phys_addr; 3166 inplace = 0; 3167 } 3168 } 3169 } else { 3170 d_offs = sym_op->cipher.data.offset; 3171 d_lens = sym_op->cipher.data.length; 3172 mc_hash_off = sym_op->cipher.data.offset + 3173 sym_op->cipher.data.length; 3174 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3175 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3176 3177 if (mc_hash_off < (sym_op->auth.data.offset + 3178 sym_op->auth.data.length)){ 3179 mc_hash_off = (sym_op->auth.data.offset + 3180 sym_op->auth.data.length); 3181 } 3182 /* for gmac, salt should be updated like in gcm */ 3183 if (unlikely(sess_misc->is_gmac)) { 3184 uint8_t *salt; 3185 salt = fc_params.iv_buf; 3186 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3187 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3188 sess_misc->salt = *(uint32_t *)salt; 3189 } 3190 fc_params.iv_buf = salt + 4; 3191 } 3192 if (likely(sess_misc->mac_len)) { 3193 struct rte_mbuf *m; 3194 3195 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3196 if (!m) 3197 m = m_src; 3198 3199 /* hmac immediately following data is best case */ 3200 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3201 mc_hash_off != 3202 (uint8_t *)sym_op->auth.digest.data)) { 3203 flags |= VALID_MAC_BUF; 3204 fc_params.mac_buf.size = 3205 sess_misc->mac_len; 3206 fc_params.mac_buf.vaddr = 3207 sym_op->auth.digest.data; 3208 fc_params.mac_buf.dma_addr = 3209 sym_op->auth.digest.phys_addr; 3210 inplace = 0; 3211 } 3212 } 3213 } 3214 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3215 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3216 3217 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3218 inplace = 0; 3219 3220 if (likely(!m_dst && inplace)) { 3221 /* Case of single buffer without AAD buf or 3222 * separate mac buf in place and 3223 * not air crypto 3224 */ 3225 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3226 3227 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3228 &fc_params, 3229 &flags))) { 3230 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3231 ret = -EINVAL; 3232 goto err_exit; 3233 } 3234 3235 } else { 3236 /* Out of place processing */ 3237 fc_params.src_iov = (void *)src; 3238 fc_params.dst_iov = (void *)dst; 3239 3240 /* Store SG I/O in the api for reuse */ 3241 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3242 CPT_LOG_DP_ERR("Prepare src iov failed"); 3243 ret = -EINVAL; 3244 goto err_exit; 3245 } 3246 3247 if (unlikely(m_dst != NULL)) { 3248 uint32_t pkt_len; 3249 3250 /* Try to make room as much as src has */ 3251 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3252 3253 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3254 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3255 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3256 CPT_LOG_DP_ERR("Not enough space in " 3257 "m_dst %p, need %u" 3258 " more", 3259 m_dst, pkt_len); 3260 ret = -EINVAL; 3261 goto err_exit; 3262 } 3263 } 3264 3265 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3266 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3267 "m_dst %p", m_dst); 3268 ret = -EINVAL; 3269 goto err_exit; 3270 } 3271 } else { 3272 fc_params.dst_iov = (void *)src; 3273 } 3274 } 3275 3276 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3277 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3278 m_info->lb_mlen, m_info->pool); 3279 else 3280 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3281 m_info->sg_mlen, m_info->pool); 3282 3283 if (unlikely(mdata == NULL)) { 3284 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3285 ret = -ENOMEM; 3286 goto err_exit; 3287 } 3288 3289 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3290 op[0] = (uintptr_t)mdata; 3291 op[1] = (uintptr_t)cop; 3292 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3293 space += 4 * sizeof(uint64_t); 3294 3295 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3296 fc_params.meta_buf.dma_addr += space; 3297 fc_params.meta_buf.size -= space; 3298 3299 /* Finally prepare the instruction */ 3300 if (cpt_op & CPT_OP_ENCODE) 3301 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3302 &fc_params, op); 3303 else 3304 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3305 &fc_params, op); 3306 3307 if (unlikely(*prep_req == NULL)) { 3308 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3309 ret = -EINVAL; 3310 goto free_mdata_and_exit; 3311 } 3312 3313 *mdata_ptr = mdata; 3314 3315 return 0; 3316 3317 free_mdata_and_exit: 3318 free_op_meta(mdata, m_info->pool); 3319 err_exit: 3320 return ret; 3321 } 3322 3323 static __rte_always_inline void 3324 compl_auth_verify(struct rte_crypto_op *op, 3325 uint8_t *gen_mac, 3326 uint64_t mac_len) 3327 { 3328 uint8_t *mac; 3329 struct rte_crypto_sym_op *sym_op = op->sym; 3330 3331 if (sym_op->auth.digest.data) 3332 mac = sym_op->auth.digest.data; 3333 else 3334 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3335 uint8_t *, 3336 sym_op->auth.data.length + 3337 sym_op->auth.data.offset); 3338 if (!mac) { 3339 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3340 return; 3341 } 3342 3343 if (memcmp(mac, gen_mac, mac_len)) 3344 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3345 else 3346 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3347 } 3348 3349 static __rte_always_inline void 3350 find_kasumif9_direction_and_length(uint8_t *src, 3351 uint32_t counter_num_bytes, 3352 uint32_t *addr_length_in_bits, 3353 uint8_t *addr_direction) 3354 { 3355 uint8_t found = 0; 3356 uint32_t pos; 3357 uint8_t last_byte; 3358 while (!found && counter_num_bytes > 0) { 3359 counter_num_bytes--; 3360 if (src[counter_num_bytes] == 0x00) 3361 continue; 3362 pos = rte_bsf32(src[counter_num_bytes]); 3363 if (pos == 7) { 3364 if (likely(counter_num_bytes > 0)) { 3365 last_byte = src[counter_num_bytes - 1]; 3366 *addr_direction = last_byte & 0x1; 3367 *addr_length_in_bits = counter_num_bytes * 8 3368 - 1; 3369 } 3370 } else { 3371 last_byte = src[counter_num_bytes]; 3372 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3373 *addr_length_in_bits = counter_num_bytes * 8 3374 + (8 - (pos + 2)); 3375 } 3376 found = 1; 3377 } 3378 } 3379 3380 /* 3381 * This handles all auth only except AES_GMAC 3382 */ 3383 static __rte_always_inline int 3384 fill_digest_params(struct rte_crypto_op *cop, 3385 struct cpt_sess_misc *sess, 3386 struct cpt_qp_meta_info *m_info, 3387 void **mdata_ptr, 3388 void **prep_req) 3389 { 3390 uint32_t space = 0; 3391 struct rte_crypto_sym_op *sym_op = cop->sym; 3392 void *mdata; 3393 phys_addr_t mphys; 3394 uint64_t *op; 3395 uint32_t auth_range_off; 3396 uint32_t flags = 0; 3397 uint64_t d_offs = 0, d_lens; 3398 struct rte_mbuf *m_src, *m_dst; 3399 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3400 uint16_t mac_len = sess->mac_len; 3401 fc_params_t params; 3402 char src[SRC_IOV_SIZE]; 3403 uint8_t iv_buf[16]; 3404 int ret; 3405 3406 memset(¶ms, 0, sizeof(fc_params_t)); 3407 3408 m_src = sym_op->m_src; 3409 3410 /* For just digest lets force mempool alloc */ 3411 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3412 m_info->pool); 3413 if (mdata == NULL) { 3414 ret = -ENOMEM; 3415 goto err_exit; 3416 } 3417 3418 mphys = params.meta_buf.dma_addr; 3419 3420 op = mdata; 3421 op[0] = (uintptr_t)mdata; 3422 op[1] = (uintptr_t)cop; 3423 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3424 space += 4 * sizeof(uint64_t); 3425 3426 auth_range_off = sym_op->auth.data.offset; 3427 3428 flags = VALID_MAC_BUF; 3429 params.src_iov = (void *)src; 3430 if (unlikely(sess->zsk_flag)) { 3431 /* 3432 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3433 * we will send pass through even for auth only case, 3434 * let MC handle it 3435 */ 3436 d_offs = auth_range_off; 3437 auth_range_off = 0; 3438 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3439 uint8_t *, sess->auth_iv_offset); 3440 if (sess->zsk_flag == K_F9) { 3441 uint32_t length_in_bits, num_bytes; 3442 uint8_t *src, direction = 0; 3443 3444 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3445 uint8_t *), 8); 3446 /* 3447 * This is kasumi f9, take direction from 3448 * source buffer 3449 */ 3450 length_in_bits = cop->sym->auth.data.length; 3451 num_bytes = (length_in_bits >> 3); 3452 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3453 find_kasumif9_direction_and_length(src, 3454 num_bytes, 3455 &length_in_bits, 3456 &direction); 3457 length_in_bits -= 64; 3458 cop->sym->auth.data.offset += 64; 3459 d_offs = cop->sym->auth.data.offset; 3460 auth_range_off = d_offs / 8; 3461 cop->sym->auth.data.length = length_in_bits; 3462 3463 /* Store it at end of auth iv */ 3464 iv_buf[8] = direction; 3465 params.auth_iv_buf = iv_buf; 3466 } 3467 } 3468 3469 d_lens = sym_op->auth.data.length; 3470 3471 params.ctx_buf.vaddr = SESS_PRIV(sess); 3472 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3473 3474 if (auth_op == CPT_OP_AUTH_GENERATE) { 3475 if (sym_op->auth.digest.data) { 3476 /* 3477 * Digest to be generated 3478 * in separate buffer 3479 */ 3480 params.mac_buf.size = 3481 sess->mac_len; 3482 params.mac_buf.vaddr = 3483 sym_op->auth.digest.data; 3484 params.mac_buf.dma_addr = 3485 sym_op->auth.digest.phys_addr; 3486 } else { 3487 uint32_t off = sym_op->auth.data.offset + 3488 sym_op->auth.data.length; 3489 int32_t dlen, space; 3490 3491 m_dst = sym_op->m_dst ? 3492 sym_op->m_dst : sym_op->m_src; 3493 dlen = rte_pktmbuf_pkt_len(m_dst); 3494 3495 space = off + mac_len - dlen; 3496 if (space > 0) 3497 if (!rte_pktmbuf_append(m_dst, space)) { 3498 CPT_LOG_DP_ERR("Failed to extend " 3499 "mbuf by %uB", space); 3500 ret = -EINVAL; 3501 goto free_mdata_and_exit; 3502 } 3503 3504 params.mac_buf.vaddr = 3505 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3506 params.mac_buf.dma_addr = 3507 rte_pktmbuf_iova_offset(m_dst, off); 3508 params.mac_buf.size = mac_len; 3509 } 3510 } else { 3511 /* Need space for storing generated mac */ 3512 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3513 params.mac_buf.dma_addr = mphys + space; 3514 params.mac_buf.size = mac_len; 3515 space += RTE_ALIGN_CEIL(mac_len, 8); 3516 op[2] = (uintptr_t)params.mac_buf.vaddr; 3517 op[3] = mac_len; 3518 } 3519 3520 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3521 params.meta_buf.dma_addr = mphys + space; 3522 params.meta_buf.size -= space; 3523 3524 /* Out of place processing */ 3525 params.src_iov = (void *)src; 3526 3527 /*Store SG I/O in the api for reuse */ 3528 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3529 CPT_LOG_DP_ERR("Prepare src iov failed"); 3530 ret = -EINVAL; 3531 goto free_mdata_and_exit; 3532 } 3533 3534 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3535 if (unlikely(*prep_req == NULL)) { 3536 ret = -EINVAL; 3537 goto free_mdata_and_exit; 3538 } 3539 3540 *mdata_ptr = mdata; 3541 3542 return 0; 3543 3544 free_mdata_and_exit: 3545 free_op_meta(mdata, m_info->pool); 3546 err_exit: 3547 return ret; 3548 } 3549 3550 #endif /*_CPT_UCODE_H_ */ 3551