1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline void 26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 27 { 28 int i, base; 29 30 for (i = 0; i < 4; i++) { 31 base = 4 * i; 32 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 33 (ck[base + 2] << 8) | (ck[base + 3]); 34 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 35 } 36 } 37 38 static __rte_always_inline void 39 cpt_fc_salt_update(void *ctx, 40 uint8_t *salt) 41 { 42 struct cpt_ctx *cpt_ctx = ctx; 43 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 44 } 45 46 static __rte_always_inline int 47 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 48 { 49 switch (key_len) { 50 case CPT_BYTE_16: 51 case CPT_BYTE_24: 52 case CPT_BYTE_32: 53 return 0; 54 default: 55 return -1; 56 } 57 } 58 59 static __rte_always_inline int 60 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 61 { 62 int fc_type = 0; 63 switch (type) { 64 case PASSTHROUGH: 65 fc_type = FC_GEN; 66 break; 67 case DES3_CBC: 68 case DES3_ECB: 69 fc_type = FC_GEN; 70 break; 71 case AES_CBC: 72 case AES_ECB: 73 case AES_CFB: 74 case AES_CTR: 75 case AES_GCM: 76 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 77 return -1; 78 fc_type = FC_GEN; 79 break; 80 case CHACHA20: 81 fc_type = FC_GEN; 82 break; 83 case AES_XTS: 84 key_len = key_len / 2; 85 if (unlikely(key_len == CPT_BYTE_24)) { 86 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 87 return -1; 88 } 89 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 90 return -1; 91 fc_type = FC_GEN; 92 break; 93 case ZUC_EEA3: 94 case SNOW3G_UEA2: 95 if (unlikely(key_len != 16)) 96 return -1; 97 /* No support for AEAD yet */ 98 if (unlikely(ctx->hash_type)) 99 return -1; 100 fc_type = ZUC_SNOW3G; 101 break; 102 case KASUMI_F8_CBC: 103 case KASUMI_F8_ECB: 104 if (unlikely(key_len != 16)) 105 return -1; 106 /* No support for AEAD yet */ 107 if (unlikely(ctx->hash_type)) 108 return -1; 109 fc_type = KASUMI; 110 break; 111 default: 112 return -1; 113 } 114 115 ctx->fc_type = fc_type; 116 return 0; 117 } 118 119 static __rte_always_inline void 120 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 121 { 122 cpt_ctx->enc_cipher = 0; 123 fctx->enc.enc_cipher = 0; 124 } 125 126 static __rte_always_inline void 127 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 128 { 129 mc_aes_type_t aes_key_type = 0; 130 switch (key_len) { 131 case CPT_BYTE_16: 132 aes_key_type = AES_128_BIT; 133 break; 134 case CPT_BYTE_24: 135 aes_key_type = AES_192_BIT; 136 break; 137 case CPT_BYTE_32: 138 aes_key_type = AES_256_BIT; 139 break; 140 default: 141 /* This should not happen */ 142 CPT_LOG_DP_ERR("Invalid AES key len"); 143 return; 144 } 145 fctx->enc.aes_key = aes_key_type; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 150 uint16_t key_len) 151 { 152 uint32_t keyx[4]; 153 cpt_ctx->snow3g = 1; 154 gen_key_snow3g(key, keyx); 155 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 156 cpt_ctx->zsk_flags = 0; 157 } 158 159 static __rte_always_inline void 160 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 161 uint16_t key_len) 162 { 163 cpt_ctx->snow3g = 0; 164 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 165 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 166 cpt_ctx->zsk_flags = 0; 167 } 168 169 static __rte_always_inline void 170 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 171 uint16_t key_len) 172 { 173 cpt_ctx->k_ecb = 1; 174 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 175 cpt_ctx->zsk_flags = 0; 176 } 177 178 static __rte_always_inline void 179 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 180 uint16_t key_len) 181 { 182 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 183 cpt_ctx->zsk_flags = 0; 184 } 185 186 static __rte_always_inline int 187 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 188 uint16_t key_len, uint8_t *salt) 189 { 190 struct cpt_ctx *cpt_ctx = ctx; 191 mc_fc_context_t *fctx = &cpt_ctx->fctx; 192 int ret; 193 194 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 195 if (unlikely(ret)) 196 return -1; 197 198 if (cpt_ctx->fc_type == FC_GEN) { 199 /* 200 * We need to always say IV is from DPTR as user can 201 * sometimes iverride IV per operation. 202 */ 203 fctx->enc.iv_source = CPT_FROM_DPTR; 204 205 if (cpt_ctx->auth_key_len > 64) 206 return -1; 207 } 208 209 switch (type) { 210 case PASSTHROUGH: 211 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 212 goto success; 213 case DES3_CBC: 214 /* CPT performs DES using 3DES with the 8B DES-key 215 * replicated 2 more times to match the 24B 3DES-key. 216 * Eg. If org. key is "0x0a 0x0b", then new key is 217 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 218 */ 219 if (key_len == 8) { 220 /* Skipping the first 8B as it will be copied 221 * in the regular code flow 222 */ 223 memcpy(fctx->enc.encr_key+key_len, key, key_len); 224 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 225 } 226 break; 227 case DES3_ECB: 228 /* For DES3_ECB IV need to be from CTX. */ 229 fctx->enc.iv_source = CPT_FROM_CTX; 230 break; 231 case AES_CBC: 232 case AES_ECB: 233 case AES_CFB: 234 case AES_CTR: 235 case CHACHA20: 236 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 237 break; 238 case AES_GCM: 239 /* Even though iv source is from dptr, 240 * aes_gcm salt is taken from ctx 241 */ 242 if (salt) { 243 memcpy(fctx->enc.encr_iv, salt, 4); 244 /* Assuming it was just salt update 245 * and nothing else 246 */ 247 if (!key) 248 goto success; 249 } 250 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 251 break; 252 case AES_XTS: 253 key_len = key_len / 2; 254 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 255 256 /* Copy key2 for XTS into ipad */ 257 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 258 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 259 break; 260 case SNOW3G_UEA2: 261 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 262 goto success; 263 case ZUC_EEA3: 264 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 265 goto success; 266 case KASUMI_F8_ECB: 267 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 268 goto success; 269 case KASUMI_F8_CBC: 270 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 271 goto success; 272 default: 273 return -1; 274 } 275 276 /* Only for FC_GEN case */ 277 278 /* For GMAC auth, cipher must be NULL */ 279 if (cpt_ctx->hash_type != GMAC_TYPE) 280 fctx->enc.enc_cipher = type; 281 282 memcpy(fctx->enc.encr_key, key, key_len); 283 284 success: 285 cpt_ctx->enc_cipher = type; 286 287 return 0; 288 } 289 290 static __rte_always_inline uint32_t 291 fill_sg_comp(sg_comp_t *list, 292 uint32_t i, 293 phys_addr_t dma_addr, 294 uint32_t size) 295 { 296 sg_comp_t *to = &list[i>>2]; 297 298 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 299 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 300 i++; 301 return i; 302 } 303 304 static __rte_always_inline uint32_t 305 fill_sg_comp_from_buf(sg_comp_t *list, 306 uint32_t i, 307 buf_ptr_t *from) 308 { 309 sg_comp_t *to = &list[i>>2]; 310 311 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 312 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 313 i++; 314 return i; 315 } 316 317 static __rte_always_inline uint32_t 318 fill_sg_comp_from_buf_min(sg_comp_t *list, 319 uint32_t i, 320 buf_ptr_t *from, 321 uint32_t *psize) 322 { 323 sg_comp_t *to = &list[i >> 2]; 324 uint32_t size = *psize; 325 uint32_t e_len; 326 327 e_len = (size > from->size) ? from->size : size; 328 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 329 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 330 *psize -= e_len; 331 i++; 332 return i; 333 } 334 335 /* 336 * This fills the MC expected SGIO list 337 * from IOV given by user. 338 */ 339 static __rte_always_inline uint32_t 340 fill_sg_comp_from_iov(sg_comp_t *list, 341 uint32_t i, 342 iov_ptr_t *from, uint32_t from_offset, 343 uint32_t *psize, buf_ptr_t *extra_buf, 344 uint32_t extra_offset) 345 { 346 int32_t j; 347 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 348 uint32_t size = *psize; 349 buf_ptr_t *bufs; 350 351 bufs = from->bufs; 352 for (j = 0; (j < from->buf_cnt) && size; j++) { 353 phys_addr_t e_dma_addr; 354 uint32_t e_len; 355 sg_comp_t *to = &list[i >> 2]; 356 357 if (unlikely(from_offset)) { 358 if (from_offset >= bufs[j].size) { 359 from_offset -= bufs[j].size; 360 continue; 361 } 362 e_dma_addr = bufs[j].dma_addr + from_offset; 363 e_len = (size > (bufs[j].size - from_offset)) ? 364 (bufs[j].size - from_offset) : size; 365 from_offset = 0; 366 } else { 367 e_dma_addr = bufs[j].dma_addr; 368 e_len = (size > bufs[j].size) ? 369 bufs[j].size : size; 370 } 371 372 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 373 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 374 375 if (extra_len && (e_len >= extra_offset)) { 376 /* Break the data at given offset */ 377 uint32_t next_len = e_len - extra_offset; 378 phys_addr_t next_dma = e_dma_addr + extra_offset; 379 380 if (!extra_offset) { 381 i--; 382 } else { 383 e_len = extra_offset; 384 size -= e_len; 385 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 386 } 387 388 extra_len = RTE_MIN(extra_len, size); 389 /* Insert extra data ptr */ 390 if (extra_len) { 391 i++; 392 to = &list[i >> 2]; 393 to->u.s.len[i % 4] = 394 rte_cpu_to_be_16(extra_len); 395 to->ptr[i % 4] = 396 rte_cpu_to_be_64(extra_buf->dma_addr); 397 size -= extra_len; 398 } 399 400 next_len = RTE_MIN(next_len, size); 401 /* insert the rest of the data */ 402 if (next_len) { 403 i++; 404 to = &list[i >> 2]; 405 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 406 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 407 size -= next_len; 408 } 409 extra_len = 0; 410 411 } else { 412 size -= e_len; 413 } 414 if (extra_offset) 415 extra_offset -= size; 416 i++; 417 } 418 419 *psize = size; 420 return (uint32_t)i; 421 } 422 423 static __rte_always_inline void 424 cpt_digest_gen_prep(uint32_t flags, 425 uint64_t d_lens, 426 digest_params_t *params, 427 void *op, 428 void **prep_req) 429 { 430 struct cpt_request_info *req; 431 uint32_t size, i; 432 uint16_t data_len, mac_len, key_len; 433 auth_type_t hash_type; 434 buf_ptr_t *meta_p; 435 struct cpt_ctx *ctx; 436 sg_comp_t *gather_comp; 437 sg_comp_t *scatter_comp; 438 uint8_t *in_buffer; 439 uint32_t g_size_bytes, s_size_bytes; 440 uint64_t dptr_dma, rptr_dma; 441 vq_cmd_word0_t vq_cmd_w0; 442 vq_cmd_word3_t vq_cmd_w3; 443 void *c_vaddr, *m_vaddr; 444 uint64_t c_dma, m_dma; 445 opcode_info_t opcode; 446 447 ctx = params->ctx_buf.vaddr; 448 meta_p = ¶ms->meta_buf; 449 450 m_vaddr = meta_p->vaddr; 451 m_dma = meta_p->dma_addr; 452 453 /* 454 * Save initial space that followed app data for completion code & 455 * alternate completion code to fall in same cache line as app data 456 */ 457 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 458 m_dma += COMPLETION_CODE_SIZE; 459 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 460 (uint8_t *)m_vaddr; 461 c_vaddr = (uint8_t *)m_vaddr + size; 462 c_dma = m_dma + size; 463 size += sizeof(cpt_res_s_t); 464 465 m_vaddr = (uint8_t *)m_vaddr + size; 466 m_dma += size; 467 468 req = m_vaddr; 469 470 size = sizeof(struct cpt_request_info); 471 m_vaddr = (uint8_t *)m_vaddr + size; 472 m_dma += size; 473 474 hash_type = ctx->hash_type; 475 mac_len = ctx->mac_len; 476 key_len = ctx->auth_key_len; 477 data_len = AUTH_DLEN(d_lens); 478 479 /*GP op header */ 480 vq_cmd_w0.u64 = 0; 481 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 482 if (ctx->hmac) { 483 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 484 vq_cmd_w0.s.param1 = key_len; 485 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 486 } else { 487 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 488 vq_cmd_w0.s.param1 = 0; 489 vq_cmd_w0.s.dlen = data_len; 490 } 491 492 opcode.s.minor = 0; 493 494 /* Null auth only case enters the if */ 495 if (unlikely(!hash_type && !ctx->enc_cipher)) { 496 opcode.s.major = CPT_MAJOR_OP_MISC; 497 /* Minor op is passthrough */ 498 opcode.s.minor = 0x03; 499 /* Send out completion code only */ 500 vq_cmd_w0.s.param2 = 0x1; 501 } 502 503 vq_cmd_w0.s.opcode = opcode.flags; 504 505 /* DPTR has SG list */ 506 in_buffer = m_vaddr; 507 dptr_dma = m_dma; 508 509 ((uint16_t *)in_buffer)[0] = 0; 510 ((uint16_t *)in_buffer)[1] = 0; 511 512 /* TODO Add error check if space will be sufficient */ 513 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 514 515 /* 516 * Input gather list 517 */ 518 519 i = 0; 520 521 if (ctx->hmac) { 522 uint64_t k_dma = params->ctx_buf.dma_addr + 523 offsetof(struct cpt_ctx, auth_key); 524 /* Key */ 525 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 526 } 527 528 /* input data */ 529 size = data_len; 530 if (size) { 531 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 532 0, &size, NULL, 0); 533 if (unlikely(size)) { 534 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 535 " by %dB", size); 536 return; 537 } 538 } else { 539 /* 540 * Looks like we need to support zero data 541 * gather ptr in case of hash & hmac 542 */ 543 i++; 544 } 545 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 546 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 547 548 /* 549 * Output Gather list 550 */ 551 552 i = 0; 553 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 554 555 if (flags & VALID_MAC_BUF) { 556 if (unlikely(params->mac_buf.size < mac_len)) { 557 CPT_LOG_DP_ERR("Insufficient MAC size"); 558 return; 559 } 560 561 size = mac_len; 562 i = fill_sg_comp_from_buf_min(scatter_comp, i, 563 ¶ms->mac_buf, &size); 564 } else { 565 size = mac_len; 566 i = fill_sg_comp_from_iov(scatter_comp, i, 567 params->src_iov, data_len, 568 &size, NULL, 0); 569 if (unlikely(size)) { 570 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 571 " %dB", size); 572 return; 573 } 574 } 575 576 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 577 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 578 579 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 580 581 /* This is DPTR len incase of SG mode */ 582 vq_cmd_w0.s.dlen = size; 583 584 m_vaddr = (uint8_t *)m_vaddr + size; 585 m_dma += size; 586 587 /* cpt alternate completion address saved earlier */ 588 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 589 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 590 rptr_dma = c_dma - 8; 591 592 req->ist.ei1 = dptr_dma; 593 req->ist.ei2 = rptr_dma; 594 595 /* vq command w3 */ 596 vq_cmd_w3.u64 = 0; 597 598 /* 16 byte aligned cpt res address */ 599 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 600 *req->completion_addr = COMPLETION_CODE_INIT; 601 req->comp_baddr = c_dma; 602 603 /* Fill microcode part of instruction */ 604 req->ist.ei0 = vq_cmd_w0.u64; 605 req->ist.ei3 = vq_cmd_w3.u64; 606 607 req->op = op; 608 609 *prep_req = req; 610 return; 611 } 612 613 static __rte_always_inline void 614 cpt_enc_hmac_prep(uint32_t flags, 615 uint64_t d_offs, 616 uint64_t d_lens, 617 fc_params_t *fc_params, 618 void *op, 619 void **prep_req) 620 { 621 uint32_t iv_offset = 0; 622 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 623 struct cpt_ctx *cpt_ctx; 624 uint32_t cipher_type, hash_type; 625 uint32_t mac_len, size; 626 uint8_t iv_len = 16; 627 struct cpt_request_info *req; 628 buf_ptr_t *meta_p, *aad_buf = NULL; 629 uint32_t encr_offset, auth_offset; 630 uint32_t encr_data_len, auth_data_len, aad_len = 0; 631 uint32_t passthrough_len = 0; 632 void *m_vaddr, *offset_vaddr; 633 uint64_t m_dma, offset_dma, ctx_dma; 634 vq_cmd_word0_t vq_cmd_w0; 635 vq_cmd_word3_t vq_cmd_w3; 636 void *c_vaddr; 637 uint64_t c_dma; 638 opcode_info_t opcode; 639 640 meta_p = &fc_params->meta_buf; 641 m_vaddr = meta_p->vaddr; 642 m_dma = meta_p->dma_addr; 643 644 encr_offset = ENCR_OFFSET(d_offs); 645 auth_offset = AUTH_OFFSET(d_offs); 646 encr_data_len = ENCR_DLEN(d_lens); 647 auth_data_len = AUTH_DLEN(d_lens); 648 if (unlikely(flags & VALID_AAD_BUF)) { 649 /* 650 * We dont support both aad 651 * and auth data separately 652 */ 653 auth_data_len = 0; 654 auth_offset = 0; 655 aad_len = fc_params->aad_buf.size; 656 aad_buf = &fc_params->aad_buf; 657 } 658 cpt_ctx = fc_params->ctx_buf.vaddr; 659 cipher_type = cpt_ctx->enc_cipher; 660 hash_type = cpt_ctx->hash_type; 661 mac_len = cpt_ctx->mac_len; 662 663 /* 664 * Save initial space that followed app data for completion code & 665 * alternate completion code to fall in same cache line as app data 666 */ 667 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 668 m_dma += COMPLETION_CODE_SIZE; 669 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 670 (uint8_t *)m_vaddr; 671 672 c_vaddr = (uint8_t *)m_vaddr + size; 673 c_dma = m_dma + size; 674 size += sizeof(cpt_res_s_t); 675 676 m_vaddr = (uint8_t *)m_vaddr + size; 677 m_dma += size; 678 679 /* start cpt request info struct at 8 byte boundary */ 680 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 681 (uint8_t *)m_vaddr; 682 683 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 684 685 size += sizeof(struct cpt_request_info); 686 m_vaddr = (uint8_t *)m_vaddr + size; 687 m_dma += size; 688 689 if (unlikely(!(flags & VALID_IV_BUF))) { 690 iv_len = 0; 691 iv_offset = ENCR_IV_OFFSET(d_offs); 692 } 693 694 if (unlikely(flags & VALID_AAD_BUF)) { 695 /* 696 * When AAD is given, data above encr_offset is pass through 697 * Since AAD is given as separate pointer and not as offset, 698 * this is a special case as we need to fragment input data 699 * into passthrough + encr_data and then insert AAD in between. 700 */ 701 if (hash_type != GMAC_TYPE) { 702 passthrough_len = encr_offset; 703 auth_offset = passthrough_len + iv_len; 704 encr_offset = passthrough_len + aad_len + iv_len; 705 auth_data_len = aad_len + encr_data_len; 706 } else { 707 passthrough_len = 16 + aad_len; 708 auth_offset = passthrough_len + iv_len; 709 auth_data_len = aad_len; 710 } 711 } else { 712 encr_offset += iv_len; 713 auth_offset += iv_len; 714 } 715 716 /* Encryption */ 717 opcode.s.major = CPT_MAJOR_OP_FC; 718 opcode.s.minor = 0; 719 720 if (hash_type == GMAC_TYPE) { 721 encr_offset = 0; 722 encr_data_len = 0; 723 } 724 725 auth_dlen = auth_offset + auth_data_len; 726 enc_dlen = encr_data_len + encr_offset; 727 if (unlikely(encr_data_len & 0xf)) { 728 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 729 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 730 else if (likely((cipher_type == AES_CBC) || 731 (cipher_type == AES_ECB))) 732 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 733 } 734 735 if (unlikely(auth_dlen > enc_dlen)) { 736 inputlen = auth_dlen; 737 outputlen = auth_dlen + mac_len; 738 } else { 739 inputlen = enc_dlen; 740 outputlen = enc_dlen + mac_len; 741 } 742 743 /* GP op header */ 744 vq_cmd_w0.u64 = 0; 745 vq_cmd_w0.s.param1 = encr_data_len; 746 vq_cmd_w0.s.param2 = auth_data_len; 747 /* 748 * In 83XX since we have a limitation of 749 * IV & Offset control word not part of instruction 750 * and need to be part of Data Buffer, we check if 751 * head room is there and then only do the Direct mode processing 752 */ 753 if (likely((flags & SINGLE_BUF_INPLACE) && 754 (flags & SINGLE_BUF_HEADTAILROOM))) { 755 void *dm_vaddr = fc_params->bufs[0].vaddr; 756 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 757 /* 758 * This flag indicates that there is 24 bytes head room and 759 * 8 bytes tail room available, so that we get to do 760 * DIRECT MODE with limitation 761 */ 762 763 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 764 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 765 766 /* DPTR */ 767 req->ist.ei1 = offset_dma; 768 /* RPTR should just exclude offset control word */ 769 req->ist.ei2 = dm_dma_addr - iv_len; 770 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 771 + outputlen - iv_len); 772 773 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 774 775 vq_cmd_w0.s.opcode = opcode.flags; 776 777 if (likely(iv_len)) { 778 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 779 + OFF_CTRL_LEN); 780 uint64_t *src = fc_params->iv_buf; 781 dest[0] = src[0]; 782 dest[1] = src[1]; 783 } 784 785 *(uint64_t *)offset_vaddr = 786 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 787 ((uint64_t)iv_offset << 8) | 788 ((uint64_t)auth_offset)); 789 790 } else { 791 uint32_t i, g_size_bytes, s_size_bytes; 792 uint64_t dptr_dma, rptr_dma; 793 sg_comp_t *gather_comp; 794 sg_comp_t *scatter_comp; 795 uint8_t *in_buffer; 796 797 /* This falls under strict SG mode */ 798 offset_vaddr = m_vaddr; 799 offset_dma = m_dma; 800 size = OFF_CTRL_LEN + iv_len; 801 802 m_vaddr = (uint8_t *)m_vaddr + size; 803 m_dma += size; 804 805 opcode.s.major |= CPT_DMA_MODE; 806 807 vq_cmd_w0.s.opcode = opcode.flags; 808 809 if (likely(iv_len)) { 810 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 811 + OFF_CTRL_LEN); 812 uint64_t *src = fc_params->iv_buf; 813 dest[0] = src[0]; 814 dest[1] = src[1]; 815 } 816 817 *(uint64_t *)offset_vaddr = 818 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 819 ((uint64_t)iv_offset << 8) | 820 ((uint64_t)auth_offset)); 821 822 /* DPTR has SG list */ 823 in_buffer = m_vaddr; 824 dptr_dma = m_dma; 825 826 ((uint16_t *)in_buffer)[0] = 0; 827 ((uint16_t *)in_buffer)[1] = 0; 828 829 /* TODO Add error check if space will be sufficient */ 830 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 831 832 /* 833 * Input Gather List 834 */ 835 836 i = 0; 837 838 /* Offset control word that includes iv */ 839 i = fill_sg_comp(gather_comp, i, offset_dma, 840 OFF_CTRL_LEN + iv_len); 841 842 /* Add input data */ 843 size = inputlen - iv_len; 844 if (likely(size)) { 845 uint32_t aad_offset = aad_len ? passthrough_len : 0; 846 847 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 848 i = fill_sg_comp_from_buf_min(gather_comp, i, 849 fc_params->bufs, 850 &size); 851 } else { 852 i = fill_sg_comp_from_iov(gather_comp, i, 853 fc_params->src_iov, 854 0, &size, 855 aad_buf, aad_offset); 856 } 857 858 if (unlikely(size)) { 859 CPT_LOG_DP_ERR("Insufficient buffer space," 860 " size %d needed", size); 861 return; 862 } 863 } 864 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 865 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 866 867 /* 868 * Output Scatter list 869 */ 870 i = 0; 871 scatter_comp = 872 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 873 874 /* Add IV */ 875 if (likely(iv_len)) { 876 i = fill_sg_comp(scatter_comp, i, 877 offset_dma + OFF_CTRL_LEN, 878 iv_len); 879 } 880 881 /* output data or output data + digest*/ 882 if (unlikely(flags & VALID_MAC_BUF)) { 883 size = outputlen - iv_len - mac_len; 884 if (size) { 885 uint32_t aad_offset = 886 aad_len ? passthrough_len : 0; 887 888 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 889 i = fill_sg_comp_from_buf_min( 890 scatter_comp, 891 i, 892 fc_params->bufs, 893 &size); 894 } else { 895 i = fill_sg_comp_from_iov(scatter_comp, 896 i, 897 fc_params->dst_iov, 898 0, 899 &size, 900 aad_buf, 901 aad_offset); 902 } 903 if (unlikely(size)) { 904 CPT_LOG_DP_ERR("Insufficient buffer" 905 " space, size %d needed", 906 size); 907 return; 908 } 909 } 910 /* mac_data */ 911 if (mac_len) { 912 i = fill_sg_comp_from_buf(scatter_comp, i, 913 &fc_params->mac_buf); 914 } 915 } else { 916 /* Output including mac */ 917 size = outputlen - iv_len; 918 if (likely(size)) { 919 uint32_t aad_offset = 920 aad_len ? passthrough_len : 0; 921 922 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 923 i = fill_sg_comp_from_buf_min( 924 scatter_comp, 925 i, 926 fc_params->bufs, 927 &size); 928 } else { 929 i = fill_sg_comp_from_iov(scatter_comp, 930 i, 931 fc_params->dst_iov, 932 0, 933 &size, 934 aad_buf, 935 aad_offset); 936 } 937 if (unlikely(size)) { 938 CPT_LOG_DP_ERR("Insufficient buffer" 939 " space, size %d needed", 940 size); 941 return; 942 } 943 } 944 } 945 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 946 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 947 948 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 949 950 /* This is DPTR len incase of SG mode */ 951 vq_cmd_w0.s.dlen = size; 952 953 m_vaddr = (uint8_t *)m_vaddr + size; 954 m_dma += size; 955 956 /* cpt alternate completion address saved earlier */ 957 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 958 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 959 rptr_dma = c_dma - 8; 960 961 req->ist.ei1 = dptr_dma; 962 req->ist.ei2 = rptr_dma; 963 } 964 965 ctx_dma = fc_params->ctx_buf.dma_addr + 966 offsetof(struct cpt_ctx, fctx); 967 /* vq command w3 */ 968 vq_cmd_w3.u64 = 0; 969 vq_cmd_w3.s.grp = 0; 970 vq_cmd_w3.s.cptr = ctx_dma; 971 972 /* 16 byte aligned cpt res address */ 973 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 974 *req->completion_addr = COMPLETION_CODE_INIT; 975 req->comp_baddr = c_dma; 976 977 /* Fill microcode part of instruction */ 978 req->ist.ei0 = vq_cmd_w0.u64; 979 req->ist.ei3 = vq_cmd_w3.u64; 980 981 req->op = op; 982 983 *prep_req = req; 984 return; 985 } 986 987 static __rte_always_inline void 988 cpt_dec_hmac_prep(uint32_t flags, 989 uint64_t d_offs, 990 uint64_t d_lens, 991 fc_params_t *fc_params, 992 void *op, 993 void **prep_req) 994 { 995 uint32_t iv_offset = 0, size; 996 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 997 struct cpt_ctx *cpt_ctx; 998 int32_t hash_type, mac_len; 999 uint8_t iv_len = 16; 1000 struct cpt_request_info *req; 1001 buf_ptr_t *meta_p, *aad_buf = NULL; 1002 uint32_t encr_offset, auth_offset; 1003 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1004 uint32_t passthrough_len = 0; 1005 void *m_vaddr, *offset_vaddr; 1006 uint64_t m_dma, offset_dma, ctx_dma; 1007 opcode_info_t opcode; 1008 vq_cmd_word0_t vq_cmd_w0; 1009 vq_cmd_word3_t vq_cmd_w3; 1010 void *c_vaddr; 1011 uint64_t c_dma; 1012 1013 meta_p = &fc_params->meta_buf; 1014 m_vaddr = meta_p->vaddr; 1015 m_dma = meta_p->dma_addr; 1016 1017 encr_offset = ENCR_OFFSET(d_offs); 1018 auth_offset = AUTH_OFFSET(d_offs); 1019 encr_data_len = ENCR_DLEN(d_lens); 1020 auth_data_len = AUTH_DLEN(d_lens); 1021 1022 if (unlikely(flags & VALID_AAD_BUF)) { 1023 /* 1024 * We dont support both aad 1025 * and auth data separately 1026 */ 1027 auth_data_len = 0; 1028 auth_offset = 0; 1029 aad_len = fc_params->aad_buf.size; 1030 aad_buf = &fc_params->aad_buf; 1031 } 1032 1033 cpt_ctx = fc_params->ctx_buf.vaddr; 1034 hash_type = cpt_ctx->hash_type; 1035 mac_len = cpt_ctx->mac_len; 1036 1037 if (unlikely(!(flags & VALID_IV_BUF))) { 1038 iv_len = 0; 1039 iv_offset = ENCR_IV_OFFSET(d_offs); 1040 } 1041 1042 if (unlikely(flags & VALID_AAD_BUF)) { 1043 /* 1044 * When AAD is given, data above encr_offset is pass through 1045 * Since AAD is given as separate pointer and not as offset, 1046 * this is a special case as we need to fragment input data 1047 * into passthrough + encr_data and then insert AAD in between. 1048 */ 1049 if (hash_type != GMAC_TYPE) { 1050 passthrough_len = encr_offset; 1051 auth_offset = passthrough_len + iv_len; 1052 encr_offset = passthrough_len + aad_len + iv_len; 1053 auth_data_len = aad_len + encr_data_len; 1054 } else { 1055 passthrough_len = 16 + aad_len; 1056 auth_offset = passthrough_len + iv_len; 1057 auth_data_len = aad_len; 1058 } 1059 } else { 1060 encr_offset += iv_len; 1061 auth_offset += iv_len; 1062 } 1063 1064 /* 1065 * Save initial space that followed app data for completion code & 1066 * alternate completion code to fall in same cache line as app data 1067 */ 1068 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1069 m_dma += COMPLETION_CODE_SIZE; 1070 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1071 (uint8_t *)m_vaddr; 1072 c_vaddr = (uint8_t *)m_vaddr + size; 1073 c_dma = m_dma + size; 1074 size += sizeof(cpt_res_s_t); 1075 1076 m_vaddr = (uint8_t *)m_vaddr + size; 1077 m_dma += size; 1078 1079 /* start cpt request info structure at 8 byte alignment */ 1080 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1081 (uint8_t *)m_vaddr; 1082 1083 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1084 1085 size += sizeof(struct cpt_request_info); 1086 m_vaddr = (uint8_t *)m_vaddr + size; 1087 m_dma += size; 1088 1089 /* Decryption */ 1090 opcode.s.major = CPT_MAJOR_OP_FC; 1091 opcode.s.minor = 1; 1092 1093 if (hash_type == GMAC_TYPE) { 1094 encr_offset = 0; 1095 encr_data_len = 0; 1096 } 1097 1098 enc_dlen = encr_offset + encr_data_len; 1099 auth_dlen = auth_offset + auth_data_len; 1100 1101 if (auth_dlen > enc_dlen) { 1102 inputlen = auth_dlen + mac_len; 1103 outputlen = auth_dlen; 1104 } else { 1105 inputlen = enc_dlen + mac_len; 1106 outputlen = enc_dlen; 1107 } 1108 1109 vq_cmd_w0.u64 = 0; 1110 vq_cmd_w0.s.param1 = encr_data_len; 1111 vq_cmd_w0.s.param2 = auth_data_len; 1112 1113 /* 1114 * In 83XX since we have a limitation of 1115 * IV & Offset control word not part of instruction 1116 * and need to be part of Data Buffer, we check if 1117 * head room is there and then only do the Direct mode processing 1118 */ 1119 if (likely((flags & SINGLE_BUF_INPLACE) && 1120 (flags & SINGLE_BUF_HEADTAILROOM))) { 1121 void *dm_vaddr = fc_params->bufs[0].vaddr; 1122 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1123 /* 1124 * This flag indicates that there is 24 bytes head room and 1125 * 8 bytes tail room available, so that we get to do 1126 * DIRECT MODE with limitation 1127 */ 1128 1129 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1130 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1131 req->ist.ei1 = offset_dma; 1132 1133 /* RPTR should just exclude offset control word */ 1134 req->ist.ei2 = dm_dma_addr - iv_len; 1135 1136 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1137 outputlen - iv_len); 1138 /* since this is decryption, 1139 * don't touch the content of 1140 * alternate ccode space as it contains 1141 * hmac. 1142 */ 1143 1144 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1145 1146 vq_cmd_w0.s.opcode = opcode.flags; 1147 1148 if (likely(iv_len)) { 1149 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1150 OFF_CTRL_LEN); 1151 uint64_t *src = fc_params->iv_buf; 1152 dest[0] = src[0]; 1153 dest[1] = src[1]; 1154 } 1155 1156 *(uint64_t *)offset_vaddr = 1157 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1158 ((uint64_t)iv_offset << 8) | 1159 ((uint64_t)auth_offset)); 1160 1161 } else { 1162 uint64_t dptr_dma, rptr_dma; 1163 uint32_t g_size_bytes, s_size_bytes; 1164 sg_comp_t *gather_comp; 1165 sg_comp_t *scatter_comp; 1166 uint8_t *in_buffer; 1167 uint8_t i = 0; 1168 1169 /* This falls under strict SG mode */ 1170 offset_vaddr = m_vaddr; 1171 offset_dma = m_dma; 1172 size = OFF_CTRL_LEN + iv_len; 1173 1174 m_vaddr = (uint8_t *)m_vaddr + size; 1175 m_dma += size; 1176 1177 opcode.s.major |= CPT_DMA_MODE; 1178 1179 vq_cmd_w0.s.opcode = opcode.flags; 1180 1181 if (likely(iv_len)) { 1182 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1183 OFF_CTRL_LEN); 1184 uint64_t *src = fc_params->iv_buf; 1185 dest[0] = src[0]; 1186 dest[1] = src[1]; 1187 } 1188 1189 *(uint64_t *)offset_vaddr = 1190 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1191 ((uint64_t)iv_offset << 8) | 1192 ((uint64_t)auth_offset)); 1193 1194 /* DPTR has SG list */ 1195 in_buffer = m_vaddr; 1196 dptr_dma = m_dma; 1197 1198 ((uint16_t *)in_buffer)[0] = 0; 1199 ((uint16_t *)in_buffer)[1] = 0; 1200 1201 /* TODO Add error check if space will be sufficient */ 1202 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1203 1204 /* 1205 * Input Gather List 1206 */ 1207 i = 0; 1208 1209 /* Offset control word that includes iv */ 1210 i = fill_sg_comp(gather_comp, i, offset_dma, 1211 OFF_CTRL_LEN + iv_len); 1212 1213 /* Add input data */ 1214 if (flags & VALID_MAC_BUF) { 1215 size = inputlen - iv_len - mac_len; 1216 if (size) { 1217 /* input data only */ 1218 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1219 i = fill_sg_comp_from_buf_min( 1220 gather_comp, i, 1221 fc_params->bufs, 1222 &size); 1223 } else { 1224 uint32_t aad_offset = aad_len ? 1225 passthrough_len : 0; 1226 1227 i = fill_sg_comp_from_iov(gather_comp, 1228 i, 1229 fc_params->src_iov, 1230 0, &size, 1231 aad_buf, 1232 aad_offset); 1233 } 1234 if (unlikely(size)) { 1235 CPT_LOG_DP_ERR("Insufficient buffer" 1236 " space, size %d needed", 1237 size); 1238 return; 1239 } 1240 } 1241 1242 /* mac data */ 1243 if (mac_len) { 1244 i = fill_sg_comp_from_buf(gather_comp, i, 1245 &fc_params->mac_buf); 1246 } 1247 } else { 1248 /* input data + mac */ 1249 size = inputlen - iv_len; 1250 if (size) { 1251 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1252 i = fill_sg_comp_from_buf_min( 1253 gather_comp, i, 1254 fc_params->bufs, 1255 &size); 1256 } else { 1257 uint32_t aad_offset = aad_len ? 1258 passthrough_len : 0; 1259 1260 if (unlikely(!fc_params->src_iov)) { 1261 CPT_LOG_DP_ERR("Bad input args"); 1262 return; 1263 } 1264 1265 i = fill_sg_comp_from_iov( 1266 gather_comp, i, 1267 fc_params->src_iov, 1268 0, &size, 1269 aad_buf, 1270 aad_offset); 1271 } 1272 1273 if (unlikely(size)) { 1274 CPT_LOG_DP_ERR("Insufficient buffer" 1275 " space, size %d needed", 1276 size); 1277 return; 1278 } 1279 } 1280 } 1281 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1282 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1283 1284 /* 1285 * Output Scatter List 1286 */ 1287 1288 i = 0; 1289 scatter_comp = 1290 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1291 1292 /* Add iv */ 1293 if (iv_len) { 1294 i = fill_sg_comp(scatter_comp, i, 1295 offset_dma + OFF_CTRL_LEN, 1296 iv_len); 1297 } 1298 1299 /* Add output data */ 1300 size = outputlen - iv_len; 1301 if (size) { 1302 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1303 /* handle single buffer here */ 1304 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1305 fc_params->bufs, 1306 &size); 1307 } else { 1308 uint32_t aad_offset = aad_len ? 1309 passthrough_len : 0; 1310 1311 if (unlikely(!fc_params->dst_iov)) { 1312 CPT_LOG_DP_ERR("Bad input args"); 1313 return; 1314 } 1315 1316 i = fill_sg_comp_from_iov(scatter_comp, i, 1317 fc_params->dst_iov, 0, 1318 &size, aad_buf, 1319 aad_offset); 1320 } 1321 1322 if (unlikely(size)) { 1323 CPT_LOG_DP_ERR("Insufficient buffer space," 1324 " size %d needed", size); 1325 return; 1326 } 1327 } 1328 1329 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1330 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1331 1332 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1333 1334 /* This is DPTR len incase of SG mode */ 1335 vq_cmd_w0.s.dlen = size; 1336 1337 m_vaddr = (uint8_t *)m_vaddr + size; 1338 m_dma += size; 1339 1340 /* cpt alternate completion address saved earlier */ 1341 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1342 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1343 rptr_dma = c_dma - 8; 1344 size += COMPLETION_CODE_SIZE; 1345 1346 req->ist.ei1 = dptr_dma; 1347 req->ist.ei2 = rptr_dma; 1348 } 1349 1350 ctx_dma = fc_params->ctx_buf.dma_addr + 1351 offsetof(struct cpt_ctx, fctx); 1352 /* vq command w3 */ 1353 vq_cmd_w3.u64 = 0; 1354 vq_cmd_w3.s.grp = 0; 1355 vq_cmd_w3.s.cptr = ctx_dma; 1356 1357 /* 16 byte aligned cpt res address */ 1358 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1359 *req->completion_addr = COMPLETION_CODE_INIT; 1360 req->comp_baddr = c_dma; 1361 1362 /* Fill microcode part of instruction */ 1363 req->ist.ei0 = vq_cmd_w0.u64; 1364 req->ist.ei3 = vq_cmd_w3.u64; 1365 1366 req->op = op; 1367 1368 *prep_req = req; 1369 return; 1370 } 1371 1372 static __rte_always_inline void 1373 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1374 uint64_t d_offs, 1375 uint64_t d_lens, 1376 fc_params_t *params, 1377 void *op, 1378 void **prep_req) 1379 { 1380 uint32_t size; 1381 int32_t inputlen, outputlen; 1382 struct cpt_ctx *cpt_ctx; 1383 uint32_t mac_len = 0; 1384 uint8_t snow3g, j; 1385 struct cpt_request_info *req; 1386 buf_ptr_t *buf_p; 1387 uint32_t encr_offset = 0, auth_offset = 0; 1388 uint32_t encr_data_len = 0, auth_data_len = 0; 1389 int flags, iv_len = 16; 1390 void *m_vaddr, *c_vaddr; 1391 uint64_t m_dma, c_dma, offset_ctrl; 1392 uint64_t *offset_vaddr, offset_dma; 1393 uint32_t *iv_s, iv[4]; 1394 vq_cmd_word0_t vq_cmd_w0; 1395 vq_cmd_word3_t vq_cmd_w3; 1396 opcode_info_t opcode; 1397 1398 buf_p = ¶ms->meta_buf; 1399 m_vaddr = buf_p->vaddr; 1400 m_dma = buf_p->dma_addr; 1401 1402 cpt_ctx = params->ctx_buf.vaddr; 1403 flags = cpt_ctx->zsk_flags; 1404 mac_len = cpt_ctx->mac_len; 1405 snow3g = cpt_ctx->snow3g; 1406 1407 /* 1408 * Save initial space that followed app data for completion code & 1409 * alternate completion code to fall in same cache line as app data 1410 */ 1411 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1412 m_dma += COMPLETION_CODE_SIZE; 1413 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1414 (uint8_t *)m_vaddr; 1415 1416 c_vaddr = (uint8_t *)m_vaddr + size; 1417 c_dma = m_dma + size; 1418 size += sizeof(cpt_res_s_t); 1419 1420 m_vaddr = (uint8_t *)m_vaddr + size; 1421 m_dma += size; 1422 1423 /* Reserve memory for cpt request info */ 1424 req = m_vaddr; 1425 1426 size = sizeof(struct cpt_request_info); 1427 m_vaddr = (uint8_t *)m_vaddr + size; 1428 m_dma += size; 1429 1430 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1431 1432 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1433 1434 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1435 (0 << 3) | (flags & 0x7)); 1436 1437 if (flags == 0x1) { 1438 /* 1439 * Microcode expects offsets in bytes 1440 * TODO: Rounding off 1441 */ 1442 auth_data_len = AUTH_DLEN(d_lens); 1443 1444 /* EIA3 or UIA2 */ 1445 auth_offset = AUTH_OFFSET(d_offs); 1446 auth_offset = auth_offset / 8; 1447 1448 /* consider iv len */ 1449 auth_offset += iv_len; 1450 1451 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1452 outputlen = mac_len; 1453 1454 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1455 1456 } else { 1457 /* EEA3 or UEA2 */ 1458 /* 1459 * Microcode expects offsets in bytes 1460 * TODO: Rounding off 1461 */ 1462 encr_data_len = ENCR_DLEN(d_lens); 1463 1464 encr_offset = ENCR_OFFSET(d_offs); 1465 encr_offset = encr_offset / 8; 1466 /* consider iv len */ 1467 encr_offset += iv_len; 1468 1469 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1470 outputlen = inputlen; 1471 1472 /* iv offset is 0 */ 1473 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1474 } 1475 1476 /* IV */ 1477 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1478 params->iv_buf; 1479 1480 if (snow3g) { 1481 /* 1482 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1483 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1484 */ 1485 1486 for (j = 0; j < 4; j++) 1487 iv[j] = iv_s[3 - j]; 1488 } else { 1489 /* ZUC doesn't need a swap */ 1490 for (j = 0; j < 4; j++) 1491 iv[j] = iv_s[j]; 1492 } 1493 1494 /* 1495 * GP op header, lengths are expected in bits. 1496 */ 1497 vq_cmd_w0.u64 = 0; 1498 vq_cmd_w0.s.param1 = encr_data_len; 1499 vq_cmd_w0.s.param2 = auth_data_len; 1500 1501 /* 1502 * In 83XX since we have a limitation of 1503 * IV & Offset control word not part of instruction 1504 * and need to be part of Data Buffer, we check if 1505 * head room is there and then only do the Direct mode processing 1506 */ 1507 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1508 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1509 void *dm_vaddr = params->bufs[0].vaddr; 1510 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1511 /* 1512 * This flag indicates that there is 24 bytes head room and 1513 * 8 bytes tail room available, so that we get to do 1514 * DIRECT MODE with limitation 1515 */ 1516 1517 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1518 OFF_CTRL_LEN - iv_len); 1519 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1520 1521 /* DPTR */ 1522 req->ist.ei1 = offset_dma; 1523 /* RPTR should just exclude offset control word */ 1524 req->ist.ei2 = dm_dma_addr - iv_len; 1525 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1526 + outputlen - iv_len); 1527 1528 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1529 1530 vq_cmd_w0.s.opcode = opcode.flags; 1531 1532 if (likely(iv_len)) { 1533 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1534 + OFF_CTRL_LEN); 1535 memcpy(iv_d, iv, 16); 1536 } 1537 1538 *offset_vaddr = offset_ctrl; 1539 } else { 1540 uint32_t i, g_size_bytes, s_size_bytes; 1541 uint64_t dptr_dma, rptr_dma; 1542 sg_comp_t *gather_comp; 1543 sg_comp_t *scatter_comp; 1544 uint8_t *in_buffer; 1545 uint32_t *iv_d; 1546 1547 /* save space for iv */ 1548 offset_vaddr = m_vaddr; 1549 offset_dma = m_dma; 1550 1551 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1552 m_dma += OFF_CTRL_LEN + iv_len; 1553 1554 opcode.s.major |= CPT_DMA_MODE; 1555 1556 vq_cmd_w0.s.opcode = opcode.flags; 1557 1558 /* DPTR has SG list */ 1559 in_buffer = m_vaddr; 1560 dptr_dma = m_dma; 1561 1562 ((uint16_t *)in_buffer)[0] = 0; 1563 ((uint16_t *)in_buffer)[1] = 0; 1564 1565 /* TODO Add error check if space will be sufficient */ 1566 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1567 1568 /* 1569 * Input Gather List 1570 */ 1571 i = 0; 1572 1573 /* Offset control word followed by iv */ 1574 1575 i = fill_sg_comp(gather_comp, i, offset_dma, 1576 OFF_CTRL_LEN + iv_len); 1577 1578 /* iv offset is 0 */ 1579 *offset_vaddr = offset_ctrl; 1580 1581 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1582 memcpy(iv_d, iv, 16); 1583 1584 /* input data */ 1585 size = inputlen - iv_len; 1586 if (size) { 1587 i = fill_sg_comp_from_iov(gather_comp, i, 1588 params->src_iov, 1589 0, &size, NULL, 0); 1590 if (unlikely(size)) { 1591 CPT_LOG_DP_ERR("Insufficient buffer space," 1592 " size %d needed", size); 1593 return; 1594 } 1595 } 1596 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1597 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1598 1599 /* 1600 * Output Scatter List 1601 */ 1602 1603 i = 0; 1604 scatter_comp = 1605 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1606 1607 if (flags == 0x1) { 1608 /* IV in SLIST only for EEA3 & UEA2 */ 1609 iv_len = 0; 1610 } 1611 1612 if (iv_len) { 1613 i = fill_sg_comp(scatter_comp, i, 1614 offset_dma + OFF_CTRL_LEN, iv_len); 1615 } 1616 1617 /* Add output data */ 1618 if (req_flags & VALID_MAC_BUF) { 1619 size = outputlen - iv_len - mac_len; 1620 if (size) { 1621 i = fill_sg_comp_from_iov(scatter_comp, i, 1622 params->dst_iov, 0, 1623 &size, NULL, 0); 1624 1625 if (unlikely(size)) { 1626 CPT_LOG_DP_ERR("Insufficient buffer space," 1627 " size %d needed", size); 1628 return; 1629 } 1630 } 1631 1632 /* mac data */ 1633 if (mac_len) { 1634 i = fill_sg_comp_from_buf(scatter_comp, i, 1635 ¶ms->mac_buf); 1636 } 1637 } else { 1638 /* Output including mac */ 1639 size = outputlen - iv_len; 1640 if (size) { 1641 i = fill_sg_comp_from_iov(scatter_comp, i, 1642 params->dst_iov, 0, 1643 &size, NULL, 0); 1644 1645 if (unlikely(size)) { 1646 CPT_LOG_DP_ERR("Insufficient buffer space," 1647 " size %d needed", size); 1648 return; 1649 } 1650 } 1651 } 1652 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1653 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1654 1655 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1656 1657 /* This is DPTR len incase of SG mode */ 1658 vq_cmd_w0.s.dlen = size; 1659 1660 m_vaddr = (uint8_t *)m_vaddr + size; 1661 m_dma += size; 1662 1663 /* cpt alternate completion address saved earlier */ 1664 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1665 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1666 rptr_dma = c_dma - 8; 1667 1668 req->ist.ei1 = dptr_dma; 1669 req->ist.ei2 = rptr_dma; 1670 } 1671 1672 /* vq command w3 */ 1673 vq_cmd_w3.u64 = 0; 1674 vq_cmd_w3.s.grp = 0; 1675 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1676 offsetof(struct cpt_ctx, zs_ctx); 1677 1678 /* 16 byte aligned cpt res address */ 1679 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1680 *req->completion_addr = COMPLETION_CODE_INIT; 1681 req->comp_baddr = c_dma; 1682 1683 /* Fill microcode part of instruction */ 1684 req->ist.ei0 = vq_cmd_w0.u64; 1685 req->ist.ei3 = vq_cmd_w3.u64; 1686 1687 req->op = op; 1688 1689 *prep_req = req; 1690 return; 1691 } 1692 1693 static __rte_always_inline void 1694 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1695 uint64_t d_offs, 1696 uint64_t d_lens, 1697 fc_params_t *params, 1698 void *op, 1699 void **prep_req) 1700 { 1701 uint32_t size; 1702 int32_t inputlen = 0, outputlen; 1703 struct cpt_ctx *cpt_ctx; 1704 uint8_t snow3g, iv_len = 16; 1705 struct cpt_request_info *req; 1706 buf_ptr_t *buf_p; 1707 uint32_t encr_offset; 1708 uint32_t encr_data_len; 1709 int flags; 1710 void *m_vaddr, *c_vaddr; 1711 uint64_t m_dma, c_dma; 1712 uint64_t *offset_vaddr, offset_dma; 1713 uint32_t *iv_s, iv[4], j; 1714 vq_cmd_word0_t vq_cmd_w0; 1715 vq_cmd_word3_t vq_cmd_w3; 1716 opcode_info_t opcode; 1717 1718 buf_p = ¶ms->meta_buf; 1719 m_vaddr = buf_p->vaddr; 1720 m_dma = buf_p->dma_addr; 1721 1722 /* 1723 * Microcode expects offsets in bytes 1724 * TODO: Rounding off 1725 */ 1726 encr_offset = ENCR_OFFSET(d_offs) / 8; 1727 encr_data_len = ENCR_DLEN(d_lens); 1728 1729 cpt_ctx = params->ctx_buf.vaddr; 1730 flags = cpt_ctx->zsk_flags; 1731 snow3g = cpt_ctx->snow3g; 1732 /* 1733 * Save initial space that followed app data for completion code & 1734 * alternate completion code to fall in same cache line as app data 1735 */ 1736 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1737 m_dma += COMPLETION_CODE_SIZE; 1738 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1739 (uint8_t *)m_vaddr; 1740 1741 c_vaddr = (uint8_t *)m_vaddr + size; 1742 c_dma = m_dma + size; 1743 size += sizeof(cpt_res_s_t); 1744 1745 m_vaddr = (uint8_t *)m_vaddr + size; 1746 m_dma += size; 1747 1748 /* Reserve memory for cpt request info */ 1749 req = m_vaddr; 1750 1751 size = sizeof(struct cpt_request_info); 1752 m_vaddr = (uint8_t *)m_vaddr + size; 1753 m_dma += size; 1754 1755 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1756 1757 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1758 1759 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1760 (0 << 3) | (flags & 0x7)); 1761 1762 /* consider iv len */ 1763 encr_offset += iv_len; 1764 1765 inputlen = encr_offset + 1766 (RTE_ALIGN(encr_data_len, 8) / 8); 1767 outputlen = inputlen; 1768 1769 /* IV */ 1770 iv_s = params->iv_buf; 1771 if (snow3g) { 1772 /* 1773 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1774 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1775 */ 1776 1777 for (j = 0; j < 4; j++) 1778 iv[j] = iv_s[3 - j]; 1779 } else { 1780 /* ZUC doesn't need a swap */ 1781 for (j = 0; j < 4; j++) 1782 iv[j] = iv_s[j]; 1783 } 1784 1785 /* 1786 * GP op header, lengths are expected in bits. 1787 */ 1788 vq_cmd_w0.u64 = 0; 1789 vq_cmd_w0.s.param1 = encr_data_len; 1790 1791 /* 1792 * In 83XX since we have a limitation of 1793 * IV & Offset control word not part of instruction 1794 * and need to be part of Data Buffer, we check if 1795 * head room is there and then only do the Direct mode processing 1796 */ 1797 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1798 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1799 void *dm_vaddr = params->bufs[0].vaddr; 1800 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1801 /* 1802 * This flag indicates that there is 24 bytes head room and 1803 * 8 bytes tail room available, so that we get to do 1804 * DIRECT MODE with limitation 1805 */ 1806 1807 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1808 OFF_CTRL_LEN - iv_len); 1809 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1810 1811 /* DPTR */ 1812 req->ist.ei1 = offset_dma; 1813 /* RPTR should just exclude offset control word */ 1814 req->ist.ei2 = dm_dma_addr - iv_len; 1815 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1816 + outputlen - iv_len); 1817 1818 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1819 1820 vq_cmd_w0.s.opcode = opcode.flags; 1821 1822 if (likely(iv_len)) { 1823 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1824 + OFF_CTRL_LEN); 1825 memcpy(iv_d, iv, 16); 1826 } 1827 1828 /* iv offset is 0 */ 1829 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1830 } else { 1831 uint32_t i, g_size_bytes, s_size_bytes; 1832 uint64_t dptr_dma, rptr_dma; 1833 sg_comp_t *gather_comp; 1834 sg_comp_t *scatter_comp; 1835 uint8_t *in_buffer; 1836 uint32_t *iv_d; 1837 1838 /* save space for offset and iv... */ 1839 offset_vaddr = m_vaddr; 1840 offset_dma = m_dma; 1841 1842 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1843 m_dma += OFF_CTRL_LEN + iv_len; 1844 1845 opcode.s.major |= CPT_DMA_MODE; 1846 1847 vq_cmd_w0.s.opcode = opcode.flags; 1848 1849 /* DPTR has SG list */ 1850 in_buffer = m_vaddr; 1851 dptr_dma = m_dma; 1852 1853 ((uint16_t *)in_buffer)[0] = 0; 1854 ((uint16_t *)in_buffer)[1] = 0; 1855 1856 /* TODO Add error check if space will be sufficient */ 1857 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1858 1859 /* 1860 * Input Gather List 1861 */ 1862 i = 0; 1863 1864 /* Offset control word */ 1865 1866 /* iv offset is 0 */ 1867 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1868 1869 i = fill_sg_comp(gather_comp, i, offset_dma, 1870 OFF_CTRL_LEN + iv_len); 1871 1872 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1873 memcpy(iv_d, iv, 16); 1874 1875 /* Add input data */ 1876 size = inputlen - iv_len; 1877 if (size) { 1878 i = fill_sg_comp_from_iov(gather_comp, i, 1879 params->src_iov, 1880 0, &size, NULL, 0); 1881 if (unlikely(size)) { 1882 CPT_LOG_DP_ERR("Insufficient buffer space," 1883 " size %d needed", size); 1884 return; 1885 } 1886 } 1887 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1888 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1889 1890 /* 1891 * Output Scatter List 1892 */ 1893 1894 i = 0; 1895 scatter_comp = 1896 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1897 1898 /* IV */ 1899 i = fill_sg_comp(scatter_comp, i, 1900 offset_dma + OFF_CTRL_LEN, 1901 iv_len); 1902 1903 /* Add output data */ 1904 size = outputlen - iv_len; 1905 if (size) { 1906 i = fill_sg_comp_from_iov(scatter_comp, i, 1907 params->dst_iov, 0, 1908 &size, NULL, 0); 1909 1910 if (unlikely(size)) { 1911 CPT_LOG_DP_ERR("Insufficient buffer space," 1912 " size %d needed", size); 1913 return; 1914 } 1915 } 1916 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1917 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1918 1919 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1920 1921 /* This is DPTR len incase of SG mode */ 1922 vq_cmd_w0.s.dlen = size; 1923 1924 m_vaddr = (uint8_t *)m_vaddr + size; 1925 m_dma += size; 1926 1927 /* cpt alternate completion address saved earlier */ 1928 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1929 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1930 rptr_dma = c_dma - 8; 1931 1932 req->ist.ei1 = dptr_dma; 1933 req->ist.ei2 = rptr_dma; 1934 } 1935 1936 /* vq command w3 */ 1937 vq_cmd_w3.u64 = 0; 1938 vq_cmd_w3.s.grp = 0; 1939 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1940 offsetof(struct cpt_ctx, zs_ctx); 1941 1942 /* 16 byte aligned cpt res address */ 1943 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1944 *req->completion_addr = COMPLETION_CODE_INIT; 1945 req->comp_baddr = c_dma; 1946 1947 /* Fill microcode part of instruction */ 1948 req->ist.ei0 = vq_cmd_w0.u64; 1949 req->ist.ei3 = vq_cmd_w3.u64; 1950 1951 req->op = op; 1952 1953 *prep_req = req; 1954 return; 1955 } 1956 1957 static __rte_always_inline void 1958 cpt_kasumi_enc_prep(uint32_t req_flags, 1959 uint64_t d_offs, 1960 uint64_t d_lens, 1961 fc_params_t *params, 1962 void *op, 1963 void **prep_req) 1964 { 1965 uint32_t size; 1966 int32_t inputlen = 0, outputlen = 0; 1967 struct cpt_ctx *cpt_ctx; 1968 uint32_t mac_len = 0; 1969 uint8_t i = 0; 1970 struct cpt_request_info *req; 1971 buf_ptr_t *buf_p; 1972 uint32_t encr_offset, auth_offset; 1973 uint32_t encr_data_len, auth_data_len; 1974 int flags; 1975 uint8_t *iv_s, *iv_d, iv_len = 8; 1976 uint8_t dir = 0; 1977 void *m_vaddr, *c_vaddr; 1978 uint64_t m_dma, c_dma; 1979 uint64_t *offset_vaddr, offset_dma; 1980 vq_cmd_word0_t vq_cmd_w0; 1981 vq_cmd_word3_t vq_cmd_w3; 1982 opcode_info_t opcode; 1983 uint8_t *in_buffer; 1984 uint32_t g_size_bytes, s_size_bytes; 1985 uint64_t dptr_dma, rptr_dma; 1986 sg_comp_t *gather_comp; 1987 sg_comp_t *scatter_comp; 1988 1989 buf_p = ¶ms->meta_buf; 1990 m_vaddr = buf_p->vaddr; 1991 m_dma = buf_p->dma_addr; 1992 1993 encr_offset = ENCR_OFFSET(d_offs) / 8; 1994 auth_offset = AUTH_OFFSET(d_offs) / 8; 1995 encr_data_len = ENCR_DLEN(d_lens); 1996 auth_data_len = AUTH_DLEN(d_lens); 1997 1998 cpt_ctx = params->ctx_buf.vaddr; 1999 flags = cpt_ctx->zsk_flags; 2000 mac_len = cpt_ctx->mac_len; 2001 2002 if (flags == 0x0) 2003 iv_s = params->iv_buf; 2004 else 2005 iv_s = params->auth_iv_buf; 2006 2007 dir = iv_s[8] & 0x1; 2008 2009 /* 2010 * Save initial space that followed app data for completion code & 2011 * alternate completion code to fall in same cache line as app data 2012 */ 2013 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2014 m_dma += COMPLETION_CODE_SIZE; 2015 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2016 (uint8_t *)m_vaddr; 2017 2018 c_vaddr = (uint8_t *)m_vaddr + size; 2019 c_dma = m_dma + size; 2020 size += sizeof(cpt_res_s_t); 2021 2022 m_vaddr = (uint8_t *)m_vaddr + size; 2023 m_dma += size; 2024 2025 /* Reserve memory for cpt request info */ 2026 req = m_vaddr; 2027 2028 size = sizeof(struct cpt_request_info); 2029 m_vaddr = (uint8_t *)m_vaddr + size; 2030 m_dma += size; 2031 2032 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2033 2034 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2035 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2036 (dir << 4) | (0 << 3) | (flags & 0x7)); 2037 2038 /* 2039 * GP op header, lengths are expected in bits. 2040 */ 2041 vq_cmd_w0.u64 = 0; 2042 vq_cmd_w0.s.param1 = encr_data_len; 2043 vq_cmd_w0.s.param2 = auth_data_len; 2044 vq_cmd_w0.s.opcode = opcode.flags; 2045 2046 /* consider iv len */ 2047 if (flags == 0x0) { 2048 encr_offset += iv_len; 2049 auth_offset += iv_len; 2050 } 2051 2052 /* save space for offset ctrl and iv */ 2053 offset_vaddr = m_vaddr; 2054 offset_dma = m_dma; 2055 2056 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2057 m_dma += OFF_CTRL_LEN + iv_len; 2058 2059 /* DPTR has SG list */ 2060 in_buffer = m_vaddr; 2061 dptr_dma = m_dma; 2062 2063 ((uint16_t *)in_buffer)[0] = 0; 2064 ((uint16_t *)in_buffer)[1] = 0; 2065 2066 /* TODO Add error check if space will be sufficient */ 2067 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2068 2069 /* 2070 * Input Gather List 2071 */ 2072 i = 0; 2073 2074 /* Offset control word followed by iv */ 2075 2076 if (flags == 0x0) { 2077 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2078 outputlen = inputlen; 2079 /* iv offset is 0 */ 2080 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2081 } else { 2082 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2083 outputlen = mac_len; 2084 /* iv offset is 0 */ 2085 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2086 } 2087 2088 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2089 2090 /* IV */ 2091 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2092 memcpy(iv_d, iv_s, iv_len); 2093 2094 /* input data */ 2095 size = inputlen - iv_len; 2096 if (size) { 2097 i = fill_sg_comp_from_iov(gather_comp, i, 2098 params->src_iov, 0, 2099 &size, NULL, 0); 2100 2101 if (unlikely(size)) { 2102 CPT_LOG_DP_ERR("Insufficient buffer space," 2103 " size %d needed", size); 2104 return; 2105 } 2106 } 2107 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2108 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2109 2110 /* 2111 * Output Scatter List 2112 */ 2113 2114 i = 0; 2115 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2116 2117 if (flags == 0x1) { 2118 /* IV in SLIST only for F8 */ 2119 iv_len = 0; 2120 } 2121 2122 /* IV */ 2123 if (iv_len) { 2124 i = fill_sg_comp(scatter_comp, i, 2125 offset_dma + OFF_CTRL_LEN, 2126 iv_len); 2127 } 2128 2129 /* Add output data */ 2130 if (req_flags & VALID_MAC_BUF) { 2131 size = outputlen - iv_len - mac_len; 2132 if (size) { 2133 i = fill_sg_comp_from_iov(scatter_comp, i, 2134 params->dst_iov, 0, 2135 &size, NULL, 0); 2136 2137 if (unlikely(size)) { 2138 CPT_LOG_DP_ERR("Insufficient buffer space," 2139 " size %d needed", size); 2140 return; 2141 } 2142 } 2143 2144 /* mac data */ 2145 if (mac_len) { 2146 i = fill_sg_comp_from_buf(scatter_comp, i, 2147 ¶ms->mac_buf); 2148 } 2149 } else { 2150 /* Output including mac */ 2151 size = outputlen - iv_len; 2152 if (size) { 2153 i = fill_sg_comp_from_iov(scatter_comp, i, 2154 params->dst_iov, 0, 2155 &size, NULL, 0); 2156 2157 if (unlikely(size)) { 2158 CPT_LOG_DP_ERR("Insufficient buffer space," 2159 " size %d needed", size); 2160 return; 2161 } 2162 } 2163 } 2164 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2165 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2166 2167 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2168 2169 /* This is DPTR len incase of SG mode */ 2170 vq_cmd_w0.s.dlen = size; 2171 2172 m_vaddr = (uint8_t *)m_vaddr + size; 2173 m_dma += size; 2174 2175 /* cpt alternate completion address saved earlier */ 2176 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2177 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2178 rptr_dma = c_dma - 8; 2179 2180 req->ist.ei1 = dptr_dma; 2181 req->ist.ei2 = rptr_dma; 2182 2183 /* vq command w3 */ 2184 vq_cmd_w3.u64 = 0; 2185 vq_cmd_w3.s.grp = 0; 2186 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2187 offsetof(struct cpt_ctx, k_ctx); 2188 2189 /* 16 byte aligned cpt res address */ 2190 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2191 *req->completion_addr = COMPLETION_CODE_INIT; 2192 req->comp_baddr = c_dma; 2193 2194 /* Fill microcode part of instruction */ 2195 req->ist.ei0 = vq_cmd_w0.u64; 2196 req->ist.ei3 = vq_cmd_w3.u64; 2197 2198 req->op = op; 2199 2200 *prep_req = req; 2201 return; 2202 } 2203 2204 static __rte_always_inline void 2205 cpt_kasumi_dec_prep(uint64_t d_offs, 2206 uint64_t d_lens, 2207 fc_params_t *params, 2208 void *op, 2209 void **prep_req) 2210 { 2211 uint32_t size; 2212 int32_t inputlen = 0, outputlen; 2213 struct cpt_ctx *cpt_ctx; 2214 uint8_t i = 0, iv_len = 8; 2215 struct cpt_request_info *req; 2216 buf_ptr_t *buf_p; 2217 uint32_t encr_offset; 2218 uint32_t encr_data_len; 2219 int flags; 2220 uint8_t dir = 0; 2221 void *m_vaddr, *c_vaddr; 2222 uint64_t m_dma, c_dma; 2223 uint64_t *offset_vaddr, offset_dma; 2224 vq_cmd_word0_t vq_cmd_w0; 2225 vq_cmd_word3_t vq_cmd_w3; 2226 opcode_info_t opcode; 2227 uint8_t *in_buffer; 2228 uint32_t g_size_bytes, s_size_bytes; 2229 uint64_t dptr_dma, rptr_dma; 2230 sg_comp_t *gather_comp; 2231 sg_comp_t *scatter_comp; 2232 2233 buf_p = ¶ms->meta_buf; 2234 m_vaddr = buf_p->vaddr; 2235 m_dma = buf_p->dma_addr; 2236 2237 encr_offset = ENCR_OFFSET(d_offs) / 8; 2238 encr_data_len = ENCR_DLEN(d_lens); 2239 2240 cpt_ctx = params->ctx_buf.vaddr; 2241 flags = cpt_ctx->zsk_flags; 2242 /* 2243 * Save initial space that followed app data for completion code & 2244 * alternate completion code to fall in same cache line as app data 2245 */ 2246 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2247 m_dma += COMPLETION_CODE_SIZE; 2248 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2249 (uint8_t *)m_vaddr; 2250 2251 c_vaddr = (uint8_t *)m_vaddr + size; 2252 c_dma = m_dma + size; 2253 size += sizeof(cpt_res_s_t); 2254 2255 m_vaddr = (uint8_t *)m_vaddr + size; 2256 m_dma += size; 2257 2258 /* Reserve memory for cpt request info */ 2259 req = m_vaddr; 2260 2261 size = sizeof(struct cpt_request_info); 2262 m_vaddr = (uint8_t *)m_vaddr + size; 2263 m_dma += size; 2264 2265 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2266 2267 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2268 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2269 (dir << 4) | (0 << 3) | (flags & 0x7)); 2270 2271 /* 2272 * GP op header, lengths are expected in bits. 2273 */ 2274 vq_cmd_w0.u64 = 0; 2275 vq_cmd_w0.s.param1 = encr_data_len; 2276 vq_cmd_w0.s.opcode = opcode.flags; 2277 2278 /* consider iv len */ 2279 encr_offset += iv_len; 2280 2281 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2282 outputlen = inputlen; 2283 2284 /* save space for offset ctrl & iv */ 2285 offset_vaddr = m_vaddr; 2286 offset_dma = m_dma; 2287 2288 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2289 m_dma += OFF_CTRL_LEN + iv_len; 2290 2291 /* DPTR has SG list */ 2292 in_buffer = m_vaddr; 2293 dptr_dma = m_dma; 2294 2295 ((uint16_t *)in_buffer)[0] = 0; 2296 ((uint16_t *)in_buffer)[1] = 0; 2297 2298 /* TODO Add error check if space will be sufficient */ 2299 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2300 2301 /* 2302 * Input Gather List 2303 */ 2304 i = 0; 2305 2306 /* Offset control word followed by iv */ 2307 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2308 2309 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2310 2311 /* IV */ 2312 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2313 params->iv_buf, iv_len); 2314 2315 /* Add input data */ 2316 size = inputlen - iv_len; 2317 if (size) { 2318 i = fill_sg_comp_from_iov(gather_comp, i, 2319 params->src_iov, 2320 0, &size, NULL, 0); 2321 if (unlikely(size)) { 2322 CPT_LOG_DP_ERR("Insufficient buffer space," 2323 " size %d needed", size); 2324 return; 2325 } 2326 } 2327 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2328 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2329 2330 /* 2331 * Output Scatter List 2332 */ 2333 2334 i = 0; 2335 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2336 2337 /* IV */ 2338 i = fill_sg_comp(scatter_comp, i, 2339 offset_dma + OFF_CTRL_LEN, 2340 iv_len); 2341 2342 /* Add output data */ 2343 size = outputlen - iv_len; 2344 if (size) { 2345 i = fill_sg_comp_from_iov(scatter_comp, i, 2346 params->dst_iov, 0, 2347 &size, NULL, 0); 2348 if (unlikely(size)) { 2349 CPT_LOG_DP_ERR("Insufficient buffer space," 2350 " size %d needed", size); 2351 return; 2352 } 2353 } 2354 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2355 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2356 2357 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2358 2359 /* This is DPTR len incase of SG mode */ 2360 vq_cmd_w0.s.dlen = size; 2361 2362 m_vaddr = (uint8_t *)m_vaddr + size; 2363 m_dma += size; 2364 2365 /* cpt alternate completion address saved earlier */ 2366 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2367 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2368 rptr_dma = c_dma - 8; 2369 2370 req->ist.ei1 = dptr_dma; 2371 req->ist.ei2 = rptr_dma; 2372 2373 /* vq command w3 */ 2374 vq_cmd_w3.u64 = 0; 2375 vq_cmd_w3.s.grp = 0; 2376 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2377 offsetof(struct cpt_ctx, k_ctx); 2378 2379 /* 16 byte aligned cpt res address */ 2380 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2381 *req->completion_addr = COMPLETION_CODE_INIT; 2382 req->comp_baddr = c_dma; 2383 2384 /* Fill microcode part of instruction */ 2385 req->ist.ei0 = vq_cmd_w0.u64; 2386 req->ist.ei3 = vq_cmd_w3.u64; 2387 2388 req->op = op; 2389 2390 *prep_req = req; 2391 return; 2392 } 2393 2394 static __rte_always_inline void * 2395 cpt_fc_dec_hmac_prep(uint32_t flags, 2396 uint64_t d_offs, 2397 uint64_t d_lens, 2398 fc_params_t *fc_params, 2399 void *op) 2400 { 2401 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2402 uint8_t fc_type; 2403 void *prep_req = NULL; 2404 2405 fc_type = ctx->fc_type; 2406 2407 if (likely(fc_type == FC_GEN)) { 2408 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2409 &prep_req); 2410 } else if (fc_type == ZUC_SNOW3G) { 2411 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2412 &prep_req); 2413 } else if (fc_type == KASUMI) { 2414 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2415 } 2416 2417 /* 2418 * For AUTH_ONLY case, 2419 * MC only supports digest generation and verification 2420 * should be done in software by memcmp() 2421 */ 2422 2423 return prep_req; 2424 } 2425 2426 static __rte_always_inline void *__rte_hot 2427 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2428 fc_params_t *fc_params, void *op) 2429 { 2430 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2431 uint8_t fc_type; 2432 void *prep_req = NULL; 2433 2434 fc_type = ctx->fc_type; 2435 2436 /* Common api for rest of the ops */ 2437 if (likely(fc_type == FC_GEN)) { 2438 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2439 &prep_req); 2440 } else if (fc_type == ZUC_SNOW3G) { 2441 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2442 &prep_req); 2443 } else if (fc_type == KASUMI) { 2444 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2445 &prep_req); 2446 } else if (fc_type == HASH_HMAC) { 2447 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2448 } 2449 2450 return prep_req; 2451 } 2452 2453 static __rte_always_inline int 2454 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2455 uint16_t key_len, uint16_t mac_len) 2456 { 2457 struct cpt_ctx *cpt_ctx = ctx; 2458 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2459 2460 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2461 uint32_t keyx[4]; 2462 2463 if (key_len != 16) 2464 return -1; 2465 /* No support for AEAD yet */ 2466 if (cpt_ctx->enc_cipher) 2467 return -1; 2468 /* For ZUC/SNOW3G/Kasumi */ 2469 switch (type) { 2470 case SNOW3G_UIA2: 2471 cpt_ctx->snow3g = 1; 2472 gen_key_snow3g(key, keyx); 2473 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2474 cpt_ctx->fc_type = ZUC_SNOW3G; 2475 cpt_ctx->zsk_flags = 0x1; 2476 break; 2477 case ZUC_EIA3: 2478 cpt_ctx->snow3g = 0; 2479 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2480 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2481 cpt_ctx->fc_type = ZUC_SNOW3G; 2482 cpt_ctx->zsk_flags = 0x1; 2483 break; 2484 case KASUMI_F9_ECB: 2485 /* Kasumi ECB mode */ 2486 cpt_ctx->k_ecb = 1; 2487 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2488 cpt_ctx->fc_type = KASUMI; 2489 cpt_ctx->zsk_flags = 0x1; 2490 break; 2491 case KASUMI_F9_CBC: 2492 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2493 cpt_ctx->fc_type = KASUMI; 2494 cpt_ctx->zsk_flags = 0x1; 2495 break; 2496 default: 2497 return -1; 2498 } 2499 cpt_ctx->mac_len = 4; 2500 cpt_ctx->hash_type = type; 2501 return 0; 2502 } 2503 2504 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2505 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2506 cpt_ctx->fc_type = HASH_HMAC; 2507 } 2508 2509 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2510 return -1; 2511 2512 /* For GMAC auth, cipher must be NULL */ 2513 if (type == GMAC_TYPE) 2514 fctx->enc.enc_cipher = 0; 2515 2516 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2517 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2518 2519 if (key_len) { 2520 cpt_ctx->hmac = 1; 2521 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2522 memcpy(cpt_ctx->auth_key, key, key_len); 2523 cpt_ctx->auth_key_len = key_len; 2524 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2525 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2526 2527 if (key_len <= 64) 2528 memcpy(fctx->hmac.opad, key, key_len); 2529 fctx->enc.auth_input_type = 1; 2530 } 2531 return 0; 2532 } 2533 2534 static __rte_always_inline int 2535 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2536 struct cpt_sess_misc *sess) 2537 { 2538 struct rte_crypto_aead_xform *aead_form; 2539 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2540 auth_type_t auth_type = 0; /* NULL Auth type */ 2541 uint32_t cipher_key_len = 0; 2542 uint8_t aes_gcm = 0; 2543 aead_form = &xform->aead; 2544 void *ctx = SESS_PRIV(sess); 2545 2546 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 2547 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2548 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2549 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) { 2550 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2551 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2552 } else { 2553 CPT_LOG_DP_ERR("Unknown aead operation\n"); 2554 return -1; 2555 } 2556 switch (aead_form->algo) { 2557 case RTE_CRYPTO_AEAD_AES_GCM: 2558 enc_type = AES_GCM; 2559 cipher_key_len = 16; 2560 aes_gcm = 1; 2561 break; 2562 case RTE_CRYPTO_AEAD_AES_CCM: 2563 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2564 aead_form->algo); 2565 return -1; 2566 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: 2567 enc_type = CHACHA20; 2568 auth_type = POLY1305; 2569 cipher_key_len = 32; 2570 sess->chacha_poly = 1; 2571 break; 2572 default: 2573 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2574 aead_form->algo); 2575 return -1; 2576 } 2577 if (aead_form->key.length < cipher_key_len) { 2578 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2579 (unsigned int long)aead_form->key.length); 2580 return -1; 2581 } 2582 sess->zsk_flag = 0; 2583 sess->aes_gcm = aes_gcm; 2584 sess->mac_len = aead_form->digest_length; 2585 sess->iv_offset = aead_form->iv.offset; 2586 sess->iv_length = aead_form->iv.length; 2587 sess->aad_length = aead_form->aad_length; 2588 2589 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2590 aead_form->key.length, NULL))) 2591 return -1; 2592 2593 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2594 aead_form->digest_length))) 2595 return -1; 2596 2597 return 0; 2598 } 2599 2600 static __rte_always_inline int 2601 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2602 struct cpt_sess_misc *sess) 2603 { 2604 struct rte_crypto_cipher_xform *c_form; 2605 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2606 uint32_t cipher_key_len = 0; 2607 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2608 2609 c_form = &xform->cipher; 2610 2611 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2612 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2613 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2614 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2615 else { 2616 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2617 return -1; 2618 } 2619 2620 switch (c_form->algo) { 2621 case RTE_CRYPTO_CIPHER_AES_CBC: 2622 enc_type = AES_CBC; 2623 cipher_key_len = 16; 2624 break; 2625 case RTE_CRYPTO_CIPHER_3DES_CBC: 2626 enc_type = DES3_CBC; 2627 cipher_key_len = 24; 2628 break; 2629 case RTE_CRYPTO_CIPHER_DES_CBC: 2630 /* DES is implemented using 3DES in hardware */ 2631 enc_type = DES3_CBC; 2632 cipher_key_len = 8; 2633 break; 2634 case RTE_CRYPTO_CIPHER_AES_CTR: 2635 enc_type = AES_CTR; 2636 cipher_key_len = 16; 2637 aes_ctr = 1; 2638 break; 2639 case RTE_CRYPTO_CIPHER_NULL: 2640 enc_type = 0; 2641 is_null = 1; 2642 break; 2643 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2644 enc_type = KASUMI_F8_ECB; 2645 cipher_key_len = 16; 2646 zsk_flag = K_F8; 2647 break; 2648 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2649 enc_type = SNOW3G_UEA2; 2650 cipher_key_len = 16; 2651 zsk_flag = ZS_EA; 2652 break; 2653 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2654 enc_type = ZUC_EEA3; 2655 cipher_key_len = 16; 2656 zsk_flag = ZS_EA; 2657 break; 2658 case RTE_CRYPTO_CIPHER_AES_XTS: 2659 enc_type = AES_XTS; 2660 cipher_key_len = 16; 2661 break; 2662 case RTE_CRYPTO_CIPHER_3DES_ECB: 2663 enc_type = DES3_ECB; 2664 cipher_key_len = 24; 2665 break; 2666 case RTE_CRYPTO_CIPHER_AES_ECB: 2667 enc_type = AES_ECB; 2668 cipher_key_len = 16; 2669 break; 2670 case RTE_CRYPTO_CIPHER_3DES_CTR: 2671 case RTE_CRYPTO_CIPHER_AES_F8: 2672 case RTE_CRYPTO_CIPHER_ARC4: 2673 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2674 c_form->algo); 2675 return -1; 2676 default: 2677 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2678 c_form->algo); 2679 return -1; 2680 } 2681 2682 if (c_form->key.length < cipher_key_len) { 2683 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2684 (unsigned long) c_form->key.length); 2685 return -1; 2686 } 2687 2688 sess->zsk_flag = zsk_flag; 2689 sess->aes_gcm = 0; 2690 sess->aes_ctr = aes_ctr; 2691 sess->iv_offset = c_form->iv.offset; 2692 sess->iv_length = c_form->iv.length; 2693 sess->is_null = is_null; 2694 2695 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2696 c_form->key.data, c_form->key.length, NULL))) 2697 return -1; 2698 2699 return 0; 2700 } 2701 2702 static __rte_always_inline int 2703 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2704 struct cpt_sess_misc *sess) 2705 { 2706 struct rte_crypto_auth_xform *a_form; 2707 auth_type_t auth_type = 0; /* NULL Auth type */ 2708 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2709 2710 a_form = &xform->auth; 2711 2712 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2713 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2714 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2715 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2716 else { 2717 CPT_LOG_DP_ERR("Unknown auth operation"); 2718 return -1; 2719 } 2720 2721 switch (a_form->algo) { 2722 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2723 /* Fall through */ 2724 case RTE_CRYPTO_AUTH_SHA1: 2725 auth_type = SHA1_TYPE; 2726 break; 2727 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2728 case RTE_CRYPTO_AUTH_SHA256: 2729 auth_type = SHA2_SHA256; 2730 break; 2731 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2732 case RTE_CRYPTO_AUTH_SHA512: 2733 auth_type = SHA2_SHA512; 2734 break; 2735 case RTE_CRYPTO_AUTH_AES_GMAC: 2736 auth_type = GMAC_TYPE; 2737 aes_gcm = 1; 2738 break; 2739 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2740 case RTE_CRYPTO_AUTH_SHA224: 2741 auth_type = SHA2_SHA224; 2742 break; 2743 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2744 case RTE_CRYPTO_AUTH_SHA384: 2745 auth_type = SHA2_SHA384; 2746 break; 2747 case RTE_CRYPTO_AUTH_MD5_HMAC: 2748 case RTE_CRYPTO_AUTH_MD5: 2749 auth_type = MD5_TYPE; 2750 break; 2751 case RTE_CRYPTO_AUTH_KASUMI_F9: 2752 auth_type = KASUMI_F9_ECB; 2753 /* 2754 * Indicate that direction needs to be taken out 2755 * from end of src 2756 */ 2757 zsk_flag = K_F9; 2758 break; 2759 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2760 auth_type = SNOW3G_UIA2; 2761 zsk_flag = ZS_IA; 2762 break; 2763 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2764 auth_type = ZUC_EIA3; 2765 zsk_flag = ZS_IA; 2766 break; 2767 case RTE_CRYPTO_AUTH_NULL: 2768 auth_type = 0; 2769 is_null = 1; 2770 break; 2771 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2772 case RTE_CRYPTO_AUTH_AES_CMAC: 2773 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2774 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2775 a_form->algo); 2776 return -1; 2777 default: 2778 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2779 a_form->algo); 2780 return -1; 2781 } 2782 2783 sess->zsk_flag = zsk_flag; 2784 sess->aes_gcm = aes_gcm; 2785 sess->mac_len = a_form->digest_length; 2786 sess->is_null = is_null; 2787 if (zsk_flag) { 2788 sess->auth_iv_offset = a_form->iv.offset; 2789 sess->auth_iv_length = a_form->iv.length; 2790 } 2791 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2792 a_form->key.data, a_form->key.length, 2793 a_form->digest_length))) 2794 return -1; 2795 2796 return 0; 2797 } 2798 2799 static __rte_always_inline int 2800 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2801 struct cpt_sess_misc *sess) 2802 { 2803 struct rte_crypto_auth_xform *a_form; 2804 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2805 auth_type_t auth_type = 0; /* NULL Auth type */ 2806 void *ctx = SESS_PRIV(sess); 2807 2808 a_form = &xform->auth; 2809 2810 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2811 sess->cpt_op |= CPT_OP_ENCODE; 2812 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2813 sess->cpt_op |= CPT_OP_DECODE; 2814 else { 2815 CPT_LOG_DP_ERR("Unknown auth operation"); 2816 return -1; 2817 } 2818 2819 switch (a_form->algo) { 2820 case RTE_CRYPTO_AUTH_AES_GMAC: 2821 enc_type = AES_GCM; 2822 auth_type = GMAC_TYPE; 2823 break; 2824 default: 2825 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2826 a_form->algo); 2827 return -1; 2828 } 2829 2830 sess->zsk_flag = 0; 2831 sess->aes_gcm = 0; 2832 sess->is_gmac = 1; 2833 sess->iv_offset = a_form->iv.offset; 2834 sess->iv_length = a_form->iv.length; 2835 sess->mac_len = a_form->digest_length; 2836 2837 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2838 a_form->key.length, NULL))) 2839 return -1; 2840 2841 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2842 a_form->digest_length))) 2843 return -1; 2844 2845 return 0; 2846 } 2847 2848 static __rte_always_inline void * 2849 alloc_op_meta(struct rte_mbuf *m_src, 2850 buf_ptr_t *buf, 2851 int32_t len, 2852 struct rte_mempool *cpt_meta_pool) 2853 { 2854 uint8_t *mdata; 2855 2856 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2857 if (likely(m_src && (m_src->nb_segs == 1))) { 2858 int32_t tailroom; 2859 phys_addr_t mphys; 2860 2861 /* Check if tailroom is sufficient to hold meta data */ 2862 tailroom = rte_pktmbuf_tailroom(m_src); 2863 if (likely(tailroom > len + 8)) { 2864 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2865 mphys = m_src->buf_physaddr + m_src->buf_len; 2866 mdata -= len; 2867 mphys -= len; 2868 buf->vaddr = mdata; 2869 buf->dma_addr = mphys; 2870 buf->size = len; 2871 /* Indicate that this is a mbuf allocated mdata */ 2872 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2873 return mdata; 2874 } 2875 } 2876 #else 2877 RTE_SET_USED(m_src); 2878 #endif 2879 2880 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2881 return NULL; 2882 2883 buf->vaddr = mdata; 2884 buf->dma_addr = rte_mempool_virt2iova(mdata); 2885 buf->size = len; 2886 2887 return mdata; 2888 } 2889 2890 /** 2891 * cpt_free_metabuf - free metabuf to mempool. 2892 * @param instance: pointer to instance. 2893 * @param objp: pointer to the metabuf. 2894 */ 2895 static __rte_always_inline void 2896 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2897 { 2898 bool nofree = ((uintptr_t)mdata & 1ull); 2899 2900 if (likely(nofree)) 2901 return; 2902 rte_mempool_put(cpt_meta_pool, mdata); 2903 } 2904 2905 static __rte_always_inline uint32_t 2906 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2907 iov_ptr_t *iovec, uint32_t start_offset) 2908 { 2909 uint16_t index = 0; 2910 void *seg_data = NULL; 2911 phys_addr_t seg_phys; 2912 int32_t seg_size = 0; 2913 2914 if (!pkt) { 2915 iovec->buf_cnt = 0; 2916 return 0; 2917 } 2918 2919 if (!start_offset) { 2920 seg_data = rte_pktmbuf_mtod(pkt, void *); 2921 seg_phys = rte_pktmbuf_mtophys(pkt); 2922 seg_size = pkt->data_len; 2923 } else { 2924 while (start_offset >= pkt->data_len) { 2925 start_offset -= pkt->data_len; 2926 pkt = pkt->next; 2927 } 2928 2929 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2930 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2931 seg_size = pkt->data_len - start_offset; 2932 if (!seg_size) 2933 return 1; 2934 } 2935 2936 /* first seg */ 2937 iovec->bufs[index].vaddr = seg_data; 2938 iovec->bufs[index].dma_addr = seg_phys; 2939 iovec->bufs[index].size = seg_size; 2940 index++; 2941 pkt = pkt->next; 2942 2943 while (unlikely(pkt != NULL)) { 2944 seg_data = rte_pktmbuf_mtod(pkt, void *); 2945 seg_phys = rte_pktmbuf_mtophys(pkt); 2946 seg_size = pkt->data_len; 2947 if (!seg_size) 2948 break; 2949 2950 iovec->bufs[index].vaddr = seg_data; 2951 iovec->bufs[index].dma_addr = seg_phys; 2952 iovec->bufs[index].size = seg_size; 2953 2954 index++; 2955 2956 pkt = pkt->next; 2957 } 2958 2959 iovec->buf_cnt = index; 2960 return 0; 2961 } 2962 2963 static __rte_always_inline uint32_t 2964 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2965 fc_params_t *param, 2966 uint32_t *flags) 2967 { 2968 uint16_t index = 0; 2969 void *seg_data = NULL; 2970 phys_addr_t seg_phys; 2971 uint32_t seg_size = 0; 2972 iov_ptr_t *iovec; 2973 2974 seg_data = rte_pktmbuf_mtod(pkt, void *); 2975 seg_phys = rte_pktmbuf_mtophys(pkt); 2976 seg_size = pkt->data_len; 2977 2978 /* first seg */ 2979 if (likely(!pkt->next)) { 2980 uint32_t headroom, tailroom; 2981 2982 *flags |= SINGLE_BUF_INPLACE; 2983 headroom = rte_pktmbuf_headroom(pkt); 2984 tailroom = rte_pktmbuf_tailroom(pkt); 2985 if (likely((headroom >= 24) && 2986 (tailroom >= 8))) { 2987 /* In 83XX this is prerequivisit for Direct mode */ 2988 *flags |= SINGLE_BUF_HEADTAILROOM; 2989 } 2990 param->bufs[0].vaddr = seg_data; 2991 param->bufs[0].dma_addr = seg_phys; 2992 param->bufs[0].size = seg_size; 2993 return 0; 2994 } 2995 iovec = param->src_iov; 2996 iovec->bufs[index].vaddr = seg_data; 2997 iovec->bufs[index].dma_addr = seg_phys; 2998 iovec->bufs[index].size = seg_size; 2999 index++; 3000 pkt = pkt->next; 3001 3002 while (unlikely(pkt != NULL)) { 3003 seg_data = rte_pktmbuf_mtod(pkt, void *); 3004 seg_phys = rte_pktmbuf_mtophys(pkt); 3005 seg_size = pkt->data_len; 3006 3007 if (!seg_size) 3008 break; 3009 3010 iovec->bufs[index].vaddr = seg_data; 3011 iovec->bufs[index].dma_addr = seg_phys; 3012 iovec->bufs[index].size = seg_size; 3013 3014 index++; 3015 3016 pkt = pkt->next; 3017 } 3018 3019 iovec->buf_cnt = index; 3020 return 0; 3021 } 3022 3023 static __rte_always_inline int 3024 fill_fc_params(struct rte_crypto_op *cop, 3025 struct cpt_sess_misc *sess_misc, 3026 struct cpt_qp_meta_info *m_info, 3027 void **mdata_ptr, 3028 void **prep_req) 3029 { 3030 uint32_t space = 0; 3031 struct rte_crypto_sym_op *sym_op = cop->sym; 3032 void *mdata = NULL; 3033 uintptr_t *op; 3034 uint32_t mc_hash_off; 3035 uint32_t flags = 0; 3036 uint64_t d_offs, d_lens; 3037 struct rte_mbuf *m_src, *m_dst; 3038 uint8_t cpt_op = sess_misc->cpt_op; 3039 #ifdef CPT_ALWAYS_USE_SG_MODE 3040 uint8_t inplace = 0; 3041 #else 3042 uint8_t inplace = 1; 3043 #endif 3044 fc_params_t fc_params; 3045 char src[SRC_IOV_SIZE]; 3046 char dst[SRC_IOV_SIZE]; 3047 uint32_t iv_buf[4]; 3048 int ret; 3049 3050 if (likely(sess_misc->iv_length)) { 3051 flags |= VALID_IV_BUF; 3052 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3053 uint8_t *, sess_misc->iv_offset); 3054 if (sess_misc->aes_ctr && 3055 unlikely(sess_misc->iv_length != 16)) { 3056 memcpy((uint8_t *)iv_buf, 3057 rte_crypto_op_ctod_offset(cop, 3058 uint8_t *, sess_misc->iv_offset), 12); 3059 iv_buf[3] = rte_cpu_to_be_32(0x1); 3060 fc_params.iv_buf = iv_buf; 3061 } 3062 } 3063 3064 if (sess_misc->zsk_flag) { 3065 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3066 uint8_t *, 3067 sess_misc->auth_iv_offset); 3068 if (sess_misc->zsk_flag != ZS_EA) 3069 inplace = 0; 3070 } 3071 m_src = sym_op->m_src; 3072 m_dst = sym_op->m_dst; 3073 3074 if (sess_misc->aes_gcm || sess_misc->chacha_poly) { 3075 uint8_t *salt; 3076 uint8_t *aad_data; 3077 uint16_t aad_len; 3078 3079 d_offs = sym_op->aead.data.offset; 3080 d_lens = sym_op->aead.data.length; 3081 mc_hash_off = sym_op->aead.data.offset + 3082 sym_op->aead.data.length; 3083 3084 aad_data = sym_op->aead.aad.data; 3085 aad_len = sess_misc->aad_length; 3086 if (likely((aad_data + aad_len) == 3087 rte_pktmbuf_mtod_offset(m_src, 3088 uint8_t *, 3089 sym_op->aead.data.offset))) { 3090 d_offs = (d_offs - aad_len) | (d_offs << 16); 3091 d_lens = (d_lens + aad_len) | (d_lens << 32); 3092 } else { 3093 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3094 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3095 fc_params.aad_buf.size = aad_len; 3096 flags |= VALID_AAD_BUF; 3097 inplace = 0; 3098 d_offs = d_offs << 16; 3099 d_lens = d_lens << 32; 3100 } 3101 3102 salt = fc_params.iv_buf; 3103 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3104 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3105 sess_misc->salt = *(uint32_t *)salt; 3106 } 3107 fc_params.iv_buf = salt + 4; 3108 if (likely(sess_misc->mac_len)) { 3109 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3110 m_src; 3111 3112 if (!m) 3113 m = m_src; 3114 3115 /* hmac immediately following data is best case */ 3116 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3117 mc_hash_off != 3118 (uint8_t *)sym_op->aead.digest.data)) { 3119 flags |= VALID_MAC_BUF; 3120 fc_params.mac_buf.size = sess_misc->mac_len; 3121 fc_params.mac_buf.vaddr = 3122 sym_op->aead.digest.data; 3123 fc_params.mac_buf.dma_addr = 3124 sym_op->aead.digest.phys_addr; 3125 inplace = 0; 3126 } 3127 } 3128 } else { 3129 d_offs = sym_op->cipher.data.offset; 3130 d_lens = sym_op->cipher.data.length; 3131 mc_hash_off = sym_op->cipher.data.offset + 3132 sym_op->cipher.data.length; 3133 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3134 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3135 3136 if (mc_hash_off < (sym_op->auth.data.offset + 3137 sym_op->auth.data.length)){ 3138 mc_hash_off = (sym_op->auth.data.offset + 3139 sym_op->auth.data.length); 3140 } 3141 /* for gmac, salt should be updated like in gcm */ 3142 if (unlikely(sess_misc->is_gmac)) { 3143 uint8_t *salt; 3144 salt = fc_params.iv_buf; 3145 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3146 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3147 sess_misc->salt = *(uint32_t *)salt; 3148 } 3149 fc_params.iv_buf = salt + 4; 3150 } 3151 if (likely(sess_misc->mac_len)) { 3152 struct rte_mbuf *m; 3153 3154 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3155 if (!m) 3156 m = m_src; 3157 3158 /* hmac immediately following data is best case */ 3159 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3160 mc_hash_off != 3161 (uint8_t *)sym_op->auth.digest.data)) { 3162 flags |= VALID_MAC_BUF; 3163 fc_params.mac_buf.size = 3164 sess_misc->mac_len; 3165 fc_params.mac_buf.vaddr = 3166 sym_op->auth.digest.data; 3167 fc_params.mac_buf.dma_addr = 3168 sym_op->auth.digest.phys_addr; 3169 inplace = 0; 3170 } 3171 } 3172 } 3173 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3174 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3175 3176 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3177 inplace = 0; 3178 3179 if (likely(!m_dst && inplace)) { 3180 /* Case of single buffer without AAD buf or 3181 * separate mac buf in place and 3182 * not air crypto 3183 */ 3184 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3185 3186 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3187 &fc_params, 3188 &flags))) { 3189 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3190 ret = -EINVAL; 3191 goto err_exit; 3192 } 3193 3194 } else { 3195 /* Out of place processing */ 3196 fc_params.src_iov = (void *)src; 3197 fc_params.dst_iov = (void *)dst; 3198 3199 /* Store SG I/O in the api for reuse */ 3200 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3201 CPT_LOG_DP_ERR("Prepare src iov failed"); 3202 ret = -EINVAL; 3203 goto err_exit; 3204 } 3205 3206 if (unlikely(m_dst != NULL)) { 3207 uint32_t pkt_len; 3208 3209 /* Try to make room as much as src has */ 3210 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3211 3212 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3213 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3214 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3215 CPT_LOG_DP_ERR("Not enough space in " 3216 "m_dst %p, need %u" 3217 " more", 3218 m_dst, pkt_len); 3219 ret = -EINVAL; 3220 goto err_exit; 3221 } 3222 } 3223 3224 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3225 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3226 "m_dst %p", m_dst); 3227 ret = -EINVAL; 3228 goto err_exit; 3229 } 3230 } else { 3231 fc_params.dst_iov = (void *)src; 3232 } 3233 } 3234 3235 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3236 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3237 m_info->lb_mlen, m_info->pool); 3238 else 3239 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3240 m_info->sg_mlen, m_info->pool); 3241 3242 if (unlikely(mdata == NULL)) { 3243 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3244 ret = -ENOMEM; 3245 goto err_exit; 3246 } 3247 3248 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3249 op[0] = (uintptr_t)mdata; 3250 op[1] = (uintptr_t)cop; 3251 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3252 space += 4 * sizeof(uint64_t); 3253 3254 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3255 fc_params.meta_buf.dma_addr += space; 3256 fc_params.meta_buf.size -= space; 3257 3258 /* Finally prepare the instruction */ 3259 if (cpt_op & CPT_OP_ENCODE) 3260 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3261 &fc_params, op); 3262 else 3263 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3264 &fc_params, op); 3265 3266 if (unlikely(*prep_req == NULL)) { 3267 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3268 ret = -EINVAL; 3269 goto free_mdata_and_exit; 3270 } 3271 3272 *mdata_ptr = mdata; 3273 3274 return 0; 3275 3276 free_mdata_and_exit: 3277 free_op_meta(mdata, m_info->pool); 3278 err_exit: 3279 return ret; 3280 } 3281 3282 static __rte_always_inline void 3283 compl_auth_verify(struct rte_crypto_op *op, 3284 uint8_t *gen_mac, 3285 uint64_t mac_len) 3286 { 3287 uint8_t *mac; 3288 struct rte_crypto_sym_op *sym_op = op->sym; 3289 3290 if (sym_op->auth.digest.data) 3291 mac = sym_op->auth.digest.data; 3292 else 3293 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3294 uint8_t *, 3295 sym_op->auth.data.length + 3296 sym_op->auth.data.offset); 3297 if (!mac) { 3298 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3299 return; 3300 } 3301 3302 if (memcmp(mac, gen_mac, mac_len)) 3303 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3304 else 3305 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3306 } 3307 3308 static __rte_always_inline void 3309 find_kasumif9_direction_and_length(uint8_t *src, 3310 uint32_t counter_num_bytes, 3311 uint32_t *addr_length_in_bits, 3312 uint8_t *addr_direction) 3313 { 3314 uint8_t found = 0; 3315 uint32_t pos; 3316 uint8_t last_byte; 3317 while (!found && counter_num_bytes > 0) { 3318 counter_num_bytes--; 3319 if (src[counter_num_bytes] == 0x00) 3320 continue; 3321 pos = rte_bsf32(src[counter_num_bytes]); 3322 if (pos == 7) { 3323 if (likely(counter_num_bytes > 0)) { 3324 last_byte = src[counter_num_bytes - 1]; 3325 *addr_direction = last_byte & 0x1; 3326 *addr_length_in_bits = counter_num_bytes * 8 3327 - 1; 3328 } 3329 } else { 3330 last_byte = src[counter_num_bytes]; 3331 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3332 *addr_length_in_bits = counter_num_bytes * 8 3333 + (8 - (pos + 2)); 3334 } 3335 found = 1; 3336 } 3337 } 3338 3339 /* 3340 * This handles all auth only except AES_GMAC 3341 */ 3342 static __rte_always_inline int 3343 fill_digest_params(struct rte_crypto_op *cop, 3344 struct cpt_sess_misc *sess, 3345 struct cpt_qp_meta_info *m_info, 3346 void **mdata_ptr, 3347 void **prep_req) 3348 { 3349 uint32_t space = 0; 3350 struct rte_crypto_sym_op *sym_op = cop->sym; 3351 void *mdata; 3352 phys_addr_t mphys; 3353 uint64_t *op; 3354 uint32_t auth_range_off; 3355 uint32_t flags = 0; 3356 uint64_t d_offs = 0, d_lens; 3357 struct rte_mbuf *m_src, *m_dst; 3358 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3359 uint16_t mac_len = sess->mac_len; 3360 fc_params_t params; 3361 char src[SRC_IOV_SIZE]; 3362 uint8_t iv_buf[16]; 3363 int ret; 3364 3365 memset(¶ms, 0, sizeof(fc_params_t)); 3366 3367 m_src = sym_op->m_src; 3368 3369 /* For just digest lets force mempool alloc */ 3370 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3371 m_info->pool); 3372 if (mdata == NULL) { 3373 ret = -ENOMEM; 3374 goto err_exit; 3375 } 3376 3377 mphys = params.meta_buf.dma_addr; 3378 3379 op = mdata; 3380 op[0] = (uintptr_t)mdata; 3381 op[1] = (uintptr_t)cop; 3382 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3383 space += 4 * sizeof(uint64_t); 3384 3385 auth_range_off = sym_op->auth.data.offset; 3386 3387 flags = VALID_MAC_BUF; 3388 params.src_iov = (void *)src; 3389 if (unlikely(sess->zsk_flag)) { 3390 /* 3391 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3392 * we will send pass through even for auth only case, 3393 * let MC handle it 3394 */ 3395 d_offs = auth_range_off; 3396 auth_range_off = 0; 3397 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3398 uint8_t *, sess->auth_iv_offset); 3399 if (sess->zsk_flag == K_F9) { 3400 uint32_t length_in_bits, num_bytes; 3401 uint8_t *src, direction = 0; 3402 3403 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3404 uint8_t *), 8); 3405 /* 3406 * This is kasumi f9, take direction from 3407 * source buffer 3408 */ 3409 length_in_bits = cop->sym->auth.data.length; 3410 num_bytes = (length_in_bits >> 3); 3411 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3412 find_kasumif9_direction_and_length(src, 3413 num_bytes, 3414 &length_in_bits, 3415 &direction); 3416 length_in_bits -= 64; 3417 cop->sym->auth.data.offset += 64; 3418 d_offs = cop->sym->auth.data.offset; 3419 auth_range_off = d_offs / 8; 3420 cop->sym->auth.data.length = length_in_bits; 3421 3422 /* Store it at end of auth iv */ 3423 iv_buf[8] = direction; 3424 params.auth_iv_buf = iv_buf; 3425 } 3426 } 3427 3428 d_lens = sym_op->auth.data.length; 3429 3430 params.ctx_buf.vaddr = SESS_PRIV(sess); 3431 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3432 3433 if (auth_op == CPT_OP_AUTH_GENERATE) { 3434 if (sym_op->auth.digest.data) { 3435 /* 3436 * Digest to be generated 3437 * in separate buffer 3438 */ 3439 params.mac_buf.size = 3440 sess->mac_len; 3441 params.mac_buf.vaddr = 3442 sym_op->auth.digest.data; 3443 params.mac_buf.dma_addr = 3444 sym_op->auth.digest.phys_addr; 3445 } else { 3446 uint32_t off = sym_op->auth.data.offset + 3447 sym_op->auth.data.length; 3448 int32_t dlen, space; 3449 3450 m_dst = sym_op->m_dst ? 3451 sym_op->m_dst : sym_op->m_src; 3452 dlen = rte_pktmbuf_pkt_len(m_dst); 3453 3454 space = off + mac_len - dlen; 3455 if (space > 0) 3456 if (!rte_pktmbuf_append(m_dst, space)) { 3457 CPT_LOG_DP_ERR("Failed to extend " 3458 "mbuf by %uB", space); 3459 ret = -EINVAL; 3460 goto free_mdata_and_exit; 3461 } 3462 3463 params.mac_buf.vaddr = 3464 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3465 params.mac_buf.dma_addr = 3466 rte_pktmbuf_mtophys_offset(m_dst, off); 3467 params.mac_buf.size = mac_len; 3468 } 3469 } else { 3470 /* Need space for storing generated mac */ 3471 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3472 params.mac_buf.dma_addr = mphys + space; 3473 params.mac_buf.size = mac_len; 3474 space += RTE_ALIGN_CEIL(mac_len, 8); 3475 op[2] = (uintptr_t)params.mac_buf.vaddr; 3476 op[3] = mac_len; 3477 } 3478 3479 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3480 params.meta_buf.dma_addr = mphys + space; 3481 params.meta_buf.size -= space; 3482 3483 /* Out of place processing */ 3484 params.src_iov = (void *)src; 3485 3486 /*Store SG I/O in the api for reuse */ 3487 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3488 CPT_LOG_DP_ERR("Prepare src iov failed"); 3489 ret = -EINVAL; 3490 goto free_mdata_and_exit; 3491 } 3492 3493 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3494 if (unlikely(*prep_req == NULL)) { 3495 ret = -EINVAL; 3496 goto free_mdata_and_exit; 3497 } 3498 3499 *mdata_ptr = mdata; 3500 3501 return 0; 3502 3503 free_mdata_and_exit: 3504 free_op_meta(mdata, m_info->pool); 3505 err_exit: 3506 return ret; 3507 } 3508 3509 #endif /*_CPT_UCODE_H_ */ 3510