1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_validate_key(cipher_type_t type, struct cpt_ctx *cpt_ctx, 93 uint16_t key_len) 94 { 95 int fc_type = 0; 96 switch (type) { 97 case PASSTHROUGH: 98 fc_type = FC_GEN; 99 break; 100 case DES3_CBC: 101 case DES3_ECB: 102 fc_type = FC_GEN; 103 break; 104 case AES_CBC: 105 case AES_ECB: 106 case AES_CFB: 107 case AES_CTR: 108 case AES_GCM: 109 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 110 return -1; 111 fc_type = FC_GEN; 112 break; 113 case AES_XTS: 114 key_len = key_len / 2; 115 if (unlikely(key_len == CPT_BYTE_24)) { 116 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 117 return -1; 118 } 119 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 120 return -1; 121 fc_type = FC_GEN; 122 break; 123 case ZUC_EEA3: 124 case SNOW3G_UEA2: 125 if (unlikely(key_len != 16)) 126 return -1; 127 /* No support for AEAD yet */ 128 if (unlikely(cpt_ctx->hash_type)) 129 return -1; 130 fc_type = ZUC_SNOW3G; 131 break; 132 case KASUMI_F8_CBC: 133 case KASUMI_F8_ECB: 134 if (unlikely(key_len != 16)) 135 return -1; 136 /* No support for AEAD yet */ 137 if (unlikely(cpt_ctx->hash_type)) 138 return -1; 139 fc_type = KASUMI; 140 break; 141 default: 142 return -1; 143 } 144 return fc_type; 145 } 146 147 static __rte_always_inline void 148 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 149 { 150 cpt_ctx->enc_cipher = 0; 151 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 152 } 153 154 static __rte_always_inline void 155 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 156 { 157 mc_aes_type_t aes_key_type = 0; 158 switch (key_len) { 159 case CPT_BYTE_16: 160 aes_key_type = AES_128_BIT; 161 break; 162 case CPT_BYTE_24: 163 aes_key_type = AES_192_BIT; 164 break; 165 case CPT_BYTE_32: 166 aes_key_type = AES_256_BIT; 167 break; 168 default: 169 /* This should not happen */ 170 CPT_LOG_DP_ERR("Invalid AES key len"); 171 return; 172 } 173 CPT_P_ENC_CTRL(fctx).aes_key = aes_key_type; 174 } 175 176 static __rte_always_inline void 177 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, uint8_t *key, 178 uint16_t key_len) 179 { 180 uint32_t keyx[4]; 181 cpt_ctx->snow3g = 1; 182 gen_key_snow3g(key, keyx); 183 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 184 cpt_ctx->fc_type = ZUC_SNOW3G; 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->fc_type = ZUC_SNOW3G; 196 cpt_ctx->zsk_flags = 0; 197 } 198 199 static __rte_always_inline void 200 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, uint8_t *key, 201 uint16_t key_len) 202 { 203 cpt_ctx->k_ecb = 1; 204 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 205 cpt_ctx->zsk_flags = 0; 206 cpt_ctx->fc_type = KASUMI; 207 } 208 209 static __rte_always_inline void 210 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, uint8_t *key, 211 uint16_t key_len) 212 { 213 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 214 cpt_ctx->zsk_flags = 0; 215 cpt_ctx->fc_type = KASUMI; 216 } 217 218 static __rte_always_inline int 219 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, uint8_t *key, 220 uint16_t key_len, uint8_t *salt) 221 { 222 struct cpt_ctx *cpt_ctx = ctx; 223 mc_fc_context_t *fctx = &cpt_ctx->fctx; 224 uint64_t *ctrl_flags = NULL; 225 int fc_type; 226 227 /* Validate key before proceeding */ 228 fc_type = cpt_fc_ciph_validate_key(type, cpt_ctx, key_len); 229 if (unlikely(fc_type == -1)) 230 return -1; 231 232 if (fc_type == FC_GEN) { 233 cpt_ctx->fc_type = FC_GEN; 234 ctrl_flags = (uint64_t *)&(fctx->enc.enc_ctrl.flags); 235 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 236 /* 237 * We need to always say IV is from DPTR as user can 238 * sometimes iverride IV per operation. 239 */ 240 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_DPTR; 241 } 242 243 switch (type) { 244 case PASSTHROUGH: 245 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 246 goto fc_success; 247 case DES3_CBC: 248 /* CPT performs DES using 3DES with the 8B DES-key 249 * replicated 2 more times to match the 24B 3DES-key. 250 * Eg. If org. key is "0x0a 0x0b", then new key is 251 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 252 */ 253 if (key_len == 8) { 254 /* Skipping the first 8B as it will be copied 255 * in the regular code flow 256 */ 257 memcpy(fctx->enc.encr_key+key_len, key, key_len); 258 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 259 } 260 break; 261 case DES3_ECB: 262 /* For DES3_ECB IV need to be from CTX. */ 263 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_CTX; 264 break; 265 case AES_CBC: 266 case AES_ECB: 267 case AES_CFB: 268 case AES_CTR: 269 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 270 break; 271 case AES_GCM: 272 /* Even though iv source is from dptr, 273 * aes_gcm salt is taken from ctx 274 */ 275 if (salt) { 276 memcpy(fctx->enc.encr_iv, salt, 4); 277 /* Assuming it was just salt update 278 * and nothing else 279 */ 280 if (!key) 281 goto fc_success; 282 } 283 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 284 break; 285 case AES_XTS: 286 key_len = key_len / 2; 287 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 288 289 /* Copy key2 for XTS into ipad */ 290 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 291 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 292 break; 293 case SNOW3G_UEA2: 294 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 295 goto success; 296 case ZUC_EEA3: 297 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 298 goto success; 299 case KASUMI_F8_ECB: 300 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 301 goto success; 302 case KASUMI_F8_CBC: 303 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 304 goto success; 305 default: 306 break; 307 } 308 309 /* Only for FC_GEN case */ 310 311 /* For GMAC auth, cipher must be NULL */ 312 if (cpt_ctx->hash_type != GMAC_TYPE) 313 CPT_P_ENC_CTRL(fctx).enc_cipher = type; 314 315 memcpy(fctx->enc.encr_key, key, key_len); 316 317 fc_success: 318 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 319 320 success: 321 cpt_ctx->enc_cipher = type; 322 323 return 0; 324 } 325 326 static __rte_always_inline uint32_t 327 fill_sg_comp(sg_comp_t *list, 328 uint32_t i, 329 phys_addr_t dma_addr, 330 uint32_t size) 331 { 332 sg_comp_t *to = &list[i>>2]; 333 334 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 335 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 336 i++; 337 return i; 338 } 339 340 static __rte_always_inline uint32_t 341 fill_sg_comp_from_buf(sg_comp_t *list, 342 uint32_t i, 343 buf_ptr_t *from) 344 { 345 sg_comp_t *to = &list[i>>2]; 346 347 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 348 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 349 i++; 350 return i; 351 } 352 353 static __rte_always_inline uint32_t 354 fill_sg_comp_from_buf_min(sg_comp_t *list, 355 uint32_t i, 356 buf_ptr_t *from, 357 uint32_t *psize) 358 { 359 sg_comp_t *to = &list[i >> 2]; 360 uint32_t size = *psize; 361 uint32_t e_len; 362 363 e_len = (size > from->size) ? from->size : size; 364 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 365 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 366 *psize -= e_len; 367 i++; 368 return i; 369 } 370 371 /* 372 * This fills the MC expected SGIO list 373 * from IOV given by user. 374 */ 375 static __rte_always_inline uint32_t 376 fill_sg_comp_from_iov(sg_comp_t *list, 377 uint32_t i, 378 iov_ptr_t *from, uint32_t from_offset, 379 uint32_t *psize, buf_ptr_t *extra_buf, 380 uint32_t extra_offset) 381 { 382 int32_t j; 383 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 384 uint32_t size = *psize - extra_len; 385 buf_ptr_t *bufs; 386 387 bufs = from->bufs; 388 for (j = 0; (j < from->buf_cnt) && size; j++) { 389 phys_addr_t e_dma_addr; 390 uint32_t e_len; 391 sg_comp_t *to = &list[i >> 2]; 392 393 if (!bufs[j].size) 394 continue; 395 396 if (unlikely(from_offset)) { 397 if (from_offset >= bufs[j].size) { 398 from_offset -= bufs[j].size; 399 continue; 400 } 401 e_dma_addr = bufs[j].dma_addr + from_offset; 402 e_len = (size > (bufs[j].size - from_offset)) ? 403 (bufs[j].size - from_offset) : size; 404 from_offset = 0; 405 } else { 406 e_dma_addr = bufs[j].dma_addr; 407 e_len = (size > bufs[j].size) ? 408 bufs[j].size : size; 409 } 410 411 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 412 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 413 414 if (extra_len && (e_len >= extra_offset)) { 415 /* Break the data at given offset */ 416 uint32_t next_len = e_len - extra_offset; 417 phys_addr_t next_dma = e_dma_addr + extra_offset; 418 419 if (!extra_offset) { 420 i--; 421 } else { 422 e_len = extra_offset; 423 size -= e_len; 424 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 425 } 426 427 /* Insert extra data ptr */ 428 if (extra_len) { 429 i++; 430 to = &list[i >> 2]; 431 to->u.s.len[i % 4] = 432 rte_cpu_to_be_16(extra_buf->size); 433 to->ptr[i % 4] = 434 rte_cpu_to_be_64(extra_buf->dma_addr); 435 436 /* size already decremented by extra len */ 437 } 438 439 /* insert the rest of the data */ 440 if (next_len) { 441 i++; 442 to = &list[i >> 2]; 443 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 444 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 445 size -= next_len; 446 } 447 extra_len = 0; 448 449 } else { 450 size -= e_len; 451 } 452 if (extra_offset) 453 extra_offset -= size; 454 i++; 455 } 456 457 *psize = size; 458 return (uint32_t)i; 459 } 460 461 static __rte_always_inline int 462 cpt_digest_gen_prep(uint32_t flags, 463 uint64_t d_lens, 464 digest_params_t *params, 465 void *op, 466 void **prep_req) 467 { 468 struct cpt_request_info *req; 469 uint32_t size, i; 470 int32_t m_size; 471 uint16_t data_len, mac_len, key_len; 472 auth_type_t hash_type; 473 buf_ptr_t *meta_p; 474 struct cpt_ctx *ctx; 475 sg_comp_t *gather_comp; 476 sg_comp_t *scatter_comp; 477 uint8_t *in_buffer; 478 uint32_t g_size_bytes, s_size_bytes; 479 uint64_t dptr_dma, rptr_dma; 480 vq_cmd_word0_t vq_cmd_w0; 481 vq_cmd_word3_t vq_cmd_w3; 482 void *c_vaddr, *m_vaddr; 483 uint64_t c_dma, m_dma; 484 opcode_info_t opcode; 485 486 if (!params || !params->ctx_buf.vaddr) 487 return ERR_BAD_INPUT_ARG; 488 489 ctx = params->ctx_buf.vaddr; 490 meta_p = ¶ms->meta_buf; 491 492 if (!meta_p->vaddr || !meta_p->dma_addr) 493 return ERR_BAD_INPUT_ARG; 494 495 if (meta_p->size < sizeof(struct cpt_request_info)) 496 return ERR_BAD_INPUT_ARG; 497 498 m_vaddr = meta_p->vaddr; 499 m_dma = meta_p->dma_addr; 500 m_size = meta_p->size; 501 502 /* 503 * Save initial space that followed app data for completion code & 504 * alternate completion code to fall in same cache line as app data 505 */ 506 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 507 m_dma += COMPLETION_CODE_SIZE; 508 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 509 (uint8_t *)m_vaddr; 510 c_vaddr = (uint8_t *)m_vaddr + size; 511 c_dma = m_dma + size; 512 size += sizeof(cpt_res_s_t); 513 514 m_vaddr = (uint8_t *)m_vaddr + size; 515 m_dma += size; 516 m_size -= size; 517 518 req = m_vaddr; 519 520 size = sizeof(struct cpt_request_info); 521 m_vaddr = (uint8_t *)m_vaddr + size; 522 m_dma += size; 523 m_size -= size; 524 525 hash_type = ctx->hash_type; 526 mac_len = ctx->mac_len; 527 key_len = ctx->auth_key_len; 528 data_len = AUTH_DLEN(d_lens); 529 530 /*GP op header */ 531 vq_cmd_w0.u64 = 0; 532 vq_cmd_w0.s.param2 = rte_cpu_to_be_16(((uint16_t)hash_type << 8)); 533 if (ctx->hmac) { 534 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 535 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(key_len); 536 vq_cmd_w0.s.dlen = 537 rte_cpu_to_be_16((data_len + ROUNDUP8(key_len))); 538 } else { 539 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 540 vq_cmd_w0.s.param1 = 0; 541 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(data_len); 542 } 543 544 opcode.s.minor = 0; 545 546 /* Null auth only case enters the if */ 547 if (unlikely(!hash_type && !ctx->enc_cipher)) { 548 opcode.s.major = CPT_MAJOR_OP_MISC; 549 /* Minor op is passthrough */ 550 opcode.s.minor = 0x03; 551 /* Send out completion code only */ 552 vq_cmd_w0.s.param2 = 0x1; 553 } 554 555 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 556 557 /* DPTR has SG list */ 558 in_buffer = m_vaddr; 559 dptr_dma = m_dma; 560 561 ((uint16_t *)in_buffer)[0] = 0; 562 ((uint16_t *)in_buffer)[1] = 0; 563 564 /* TODO Add error check if space will be sufficient */ 565 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 566 567 /* 568 * Input gather list 569 */ 570 571 i = 0; 572 573 if (ctx->hmac) { 574 uint64_t k_dma = params->ctx_buf.dma_addr + 575 offsetof(struct cpt_ctx, auth_key); 576 /* Key */ 577 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 578 } 579 580 /* input data */ 581 size = data_len; 582 if (size) { 583 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 584 0, &size, NULL, 0); 585 if (size) { 586 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 587 " by %dB", size); 588 return ERR_BAD_INPUT_ARG; 589 } 590 } else { 591 /* 592 * Looks like we need to support zero data 593 * gather ptr in case of hash & hmac 594 */ 595 i++; 596 } 597 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 598 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 599 600 /* 601 * Output Gather list 602 */ 603 604 i = 0; 605 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 606 607 if (flags & VALID_MAC_BUF) { 608 if (params->mac_buf.size < mac_len) 609 return ERR_BAD_INPUT_ARG; 610 611 size = mac_len; 612 i = fill_sg_comp_from_buf_min(scatter_comp, i, 613 ¶ms->mac_buf, &size); 614 } else { 615 size = mac_len; 616 i = fill_sg_comp_from_iov(scatter_comp, i, 617 params->src_iov, data_len, 618 &size, NULL, 0); 619 if (size) { 620 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short by" 621 " %dB", size); 622 return ERR_BAD_INPUT_ARG; 623 } 624 } 625 626 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 627 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 628 629 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 630 631 /* This is DPTR len incase of SG mode */ 632 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 633 634 m_vaddr = (uint8_t *)m_vaddr + size; 635 m_dma += size; 636 m_size -= size; 637 638 /* cpt alternate completion address saved earlier */ 639 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 640 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 641 rptr_dma = c_dma - 8; 642 643 req->ist.ei1 = dptr_dma; 644 req->ist.ei2 = rptr_dma; 645 /* First 16-bit swap then 64-bit swap */ 646 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 647 * to eliminate all the swapping 648 */ 649 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 650 651 /* vq command w3 */ 652 vq_cmd_w3.u64 = 0; 653 654 /* 16 byte aligned cpt res address */ 655 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 656 *req->completion_addr = COMPLETION_CODE_INIT; 657 req->comp_baddr = c_dma; 658 659 /* Fill microcode part of instruction */ 660 req->ist.ei0 = vq_cmd_w0.u64; 661 req->ist.ei3 = vq_cmd_w3.u64; 662 663 req->op = op; 664 665 *prep_req = req; 666 return 0; 667 } 668 669 static __rte_always_inline int 670 cpt_enc_hmac_prep(uint32_t flags, 671 uint64_t d_offs, 672 uint64_t d_lens, 673 fc_params_t *fc_params, 674 void *op, 675 void **prep_req) 676 { 677 uint32_t iv_offset = 0; 678 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 679 struct cpt_ctx *cpt_ctx; 680 uint32_t cipher_type, hash_type; 681 uint32_t mac_len, size; 682 uint8_t iv_len = 16; 683 struct cpt_request_info *req; 684 buf_ptr_t *meta_p, *aad_buf = NULL; 685 uint32_t encr_offset, auth_offset; 686 uint32_t encr_data_len, auth_data_len, aad_len = 0; 687 uint32_t passthrough_len = 0; 688 void *m_vaddr, *offset_vaddr; 689 uint64_t m_dma, offset_dma, ctx_dma; 690 vq_cmd_word0_t vq_cmd_w0; 691 vq_cmd_word3_t vq_cmd_w3; 692 void *c_vaddr; 693 uint64_t c_dma; 694 int32_t m_size; 695 opcode_info_t opcode; 696 697 meta_p = &fc_params->meta_buf; 698 m_vaddr = meta_p->vaddr; 699 m_dma = meta_p->dma_addr; 700 m_size = meta_p->size; 701 702 encr_offset = ENCR_OFFSET(d_offs); 703 auth_offset = AUTH_OFFSET(d_offs); 704 encr_data_len = ENCR_DLEN(d_lens); 705 auth_data_len = AUTH_DLEN(d_lens); 706 if (unlikely(flags & VALID_AAD_BUF)) { 707 /* 708 * We dont support both aad 709 * and auth data separately 710 */ 711 auth_data_len = 0; 712 auth_offset = 0; 713 aad_len = fc_params->aad_buf.size; 714 aad_buf = &fc_params->aad_buf; 715 } 716 cpt_ctx = fc_params->ctx_buf.vaddr; 717 cipher_type = cpt_ctx->enc_cipher; 718 hash_type = cpt_ctx->hash_type; 719 mac_len = cpt_ctx->mac_len; 720 721 /* 722 * Save initial space that followed app data for completion code & 723 * alternate completion code to fall in same cache line as app data 724 */ 725 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 726 m_dma += COMPLETION_CODE_SIZE; 727 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 728 (uint8_t *)m_vaddr; 729 730 c_vaddr = (uint8_t *)m_vaddr + size; 731 c_dma = m_dma + size; 732 size += sizeof(cpt_res_s_t); 733 734 m_vaddr = (uint8_t *)m_vaddr + size; 735 m_dma += size; 736 m_size -= size; 737 738 /* start cpt request info struct at 8 byte boundary */ 739 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 740 (uint8_t *)m_vaddr; 741 742 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 743 744 size += sizeof(struct cpt_request_info); 745 m_vaddr = (uint8_t *)m_vaddr + size; 746 m_dma += size; 747 m_size -= size; 748 749 if (hash_type == GMAC_TYPE) 750 encr_data_len = 0; 751 752 if (unlikely(!(flags & VALID_IV_BUF))) { 753 iv_len = 0; 754 iv_offset = ENCR_IV_OFFSET(d_offs); 755 } 756 757 if (unlikely(flags & VALID_AAD_BUF)) { 758 /* 759 * When AAD is given, data above encr_offset is pass through 760 * Since AAD is given as separate pointer and not as offset, 761 * this is a special case as we need to fragment input data 762 * into passthrough + encr_data and then insert AAD in between. 763 */ 764 if (hash_type != GMAC_TYPE) { 765 passthrough_len = encr_offset; 766 auth_offset = passthrough_len + iv_len; 767 encr_offset = passthrough_len + aad_len + iv_len; 768 auth_data_len = aad_len + encr_data_len; 769 } else { 770 passthrough_len = 16 + aad_len; 771 auth_offset = passthrough_len + iv_len; 772 auth_data_len = aad_len; 773 } 774 } else { 775 encr_offset += iv_len; 776 auth_offset += iv_len; 777 } 778 779 /* Encryption */ 780 opcode.s.major = CPT_MAJOR_OP_FC; 781 opcode.s.minor = 0; 782 783 auth_dlen = auth_offset + auth_data_len; 784 enc_dlen = encr_data_len + encr_offset; 785 if (unlikely(encr_data_len & 0xf)) { 786 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 787 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 788 else if (likely((cipher_type == AES_CBC) || 789 (cipher_type == AES_ECB))) 790 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 791 } 792 793 if (unlikely(hash_type == GMAC_TYPE)) { 794 encr_offset = auth_dlen; 795 enc_dlen = 0; 796 } 797 798 if (unlikely(auth_dlen > enc_dlen)) { 799 inputlen = auth_dlen; 800 outputlen = auth_dlen + mac_len; 801 } else { 802 inputlen = enc_dlen; 803 outputlen = enc_dlen + mac_len; 804 } 805 806 /* GP op header */ 807 vq_cmd_w0.u64 = 0; 808 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len); 809 vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len); 810 /* 811 * In 83XX since we have a limitation of 812 * IV & Offset control word not part of instruction 813 * and need to be part of Data Buffer, we check if 814 * head room is there and then only do the Direct mode processing 815 */ 816 if (likely((flags & SINGLE_BUF_INPLACE) && 817 (flags & SINGLE_BUF_HEADTAILROOM))) { 818 void *dm_vaddr = fc_params->bufs[0].vaddr; 819 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 820 /* 821 * This flag indicates that there is 24 bytes head room and 822 * 8 bytes tail room available, so that we get to do 823 * DIRECT MODE with limitation 824 */ 825 826 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 827 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 828 829 /* DPTR */ 830 req->ist.ei1 = offset_dma; 831 /* RPTR should just exclude offset control word */ 832 req->ist.ei2 = dm_dma_addr - iv_len; 833 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 834 + outputlen - iv_len); 835 836 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN); 837 838 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 839 840 if (likely(iv_len)) { 841 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 842 + OFF_CTRL_LEN); 843 uint64_t *src = fc_params->iv_buf; 844 dest[0] = src[0]; 845 dest[1] = src[1]; 846 } 847 848 *(uint64_t *)offset_vaddr = 849 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 850 ((uint64_t)iv_offset << 8) | 851 ((uint64_t)auth_offset)); 852 853 } else { 854 uint32_t i, g_size_bytes, s_size_bytes; 855 uint64_t dptr_dma, rptr_dma; 856 sg_comp_t *gather_comp; 857 sg_comp_t *scatter_comp; 858 uint8_t *in_buffer; 859 860 /* This falls under strict SG mode */ 861 offset_vaddr = m_vaddr; 862 offset_dma = m_dma; 863 size = OFF_CTRL_LEN + iv_len; 864 865 m_vaddr = (uint8_t *)m_vaddr + size; 866 m_dma += size; 867 m_size -= size; 868 869 opcode.s.major |= CPT_DMA_MODE; 870 871 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 872 873 if (likely(iv_len)) { 874 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 875 + OFF_CTRL_LEN); 876 uint64_t *src = fc_params->iv_buf; 877 dest[0] = src[0]; 878 dest[1] = src[1]; 879 } 880 881 *(uint64_t *)offset_vaddr = 882 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 883 ((uint64_t)iv_offset << 8) | 884 ((uint64_t)auth_offset)); 885 886 /* DPTR has SG list */ 887 in_buffer = m_vaddr; 888 dptr_dma = m_dma; 889 890 ((uint16_t *)in_buffer)[0] = 0; 891 ((uint16_t *)in_buffer)[1] = 0; 892 893 /* TODO Add error check if space will be sufficient */ 894 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 895 896 /* 897 * Input Gather List 898 */ 899 900 i = 0; 901 902 /* Offset control word that includes iv */ 903 i = fill_sg_comp(gather_comp, i, offset_dma, 904 OFF_CTRL_LEN + iv_len); 905 906 /* Add input data */ 907 size = inputlen - iv_len; 908 if (likely(size)) { 909 uint32_t aad_offset = aad_len ? passthrough_len : 0; 910 911 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 912 i = fill_sg_comp_from_buf_min(gather_comp, i, 913 fc_params->bufs, 914 &size); 915 } else { 916 i = fill_sg_comp_from_iov(gather_comp, i, 917 fc_params->src_iov, 918 0, &size, 919 aad_buf, aad_offset); 920 } 921 922 if (unlikely(size)) { 923 CPT_LOG_DP_ERR("Insufficient buffer space," 924 " size %d needed", size); 925 return ERR_BAD_INPUT_ARG; 926 } 927 } 928 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 929 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 930 931 /* 932 * Output Scatter list 933 */ 934 i = 0; 935 scatter_comp = 936 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 937 938 /* Add IV */ 939 if (likely(iv_len)) { 940 i = fill_sg_comp(scatter_comp, i, 941 offset_dma + OFF_CTRL_LEN, 942 iv_len); 943 } 944 945 /* output data or output data + digest*/ 946 if (unlikely(flags & VALID_MAC_BUF)) { 947 size = outputlen - iv_len - mac_len; 948 if (size) { 949 uint32_t aad_offset = 950 aad_len ? passthrough_len : 0; 951 952 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 953 i = fill_sg_comp_from_buf_min( 954 scatter_comp, 955 i, 956 fc_params->bufs, 957 &size); 958 } else { 959 i = fill_sg_comp_from_iov(scatter_comp, 960 i, 961 fc_params->dst_iov, 962 0, 963 &size, 964 aad_buf, 965 aad_offset); 966 } 967 if (size) 968 return ERR_BAD_INPUT_ARG; 969 } 970 /* mac_data */ 971 if (mac_len) { 972 i = fill_sg_comp_from_buf(scatter_comp, i, 973 &fc_params->mac_buf); 974 } 975 } else { 976 /* Output including mac */ 977 size = outputlen - iv_len; 978 if (likely(size)) { 979 uint32_t aad_offset = 980 aad_len ? passthrough_len : 0; 981 982 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 983 i = fill_sg_comp_from_buf_min( 984 scatter_comp, 985 i, 986 fc_params->bufs, 987 &size); 988 } else { 989 i = fill_sg_comp_from_iov(scatter_comp, 990 i, 991 fc_params->dst_iov, 992 0, 993 &size, 994 aad_buf, 995 aad_offset); 996 } 997 if (unlikely(size)) { 998 CPT_LOG_DP_ERR("Insufficient buffer" 999 " space, size %d needed", 1000 size); 1001 return ERR_BAD_INPUT_ARG; 1002 } 1003 } 1004 } 1005 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1006 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1007 1008 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1009 1010 /* This is DPTR len incase of SG mode */ 1011 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 1012 1013 m_vaddr = (uint8_t *)m_vaddr + size; 1014 m_dma += size; 1015 m_size -= size; 1016 1017 /* cpt alternate completion address saved earlier */ 1018 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1019 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1020 rptr_dma = c_dma - 8; 1021 1022 req->ist.ei1 = dptr_dma; 1023 req->ist.ei2 = rptr_dma; 1024 } 1025 1026 /* First 16-bit swap then 64-bit swap */ 1027 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 1028 * to eliminate all the swapping 1029 */ 1030 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 1031 1032 ctx_dma = fc_params->ctx_buf.dma_addr + 1033 offsetof(struct cpt_ctx, fctx); 1034 /* vq command w3 */ 1035 vq_cmd_w3.u64 = 0; 1036 vq_cmd_w3.s.grp = 0; 1037 vq_cmd_w3.s.cptr = ctx_dma; 1038 1039 /* 16 byte aligned cpt res address */ 1040 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1041 *req->completion_addr = COMPLETION_CODE_INIT; 1042 req->comp_baddr = c_dma; 1043 1044 /* Fill microcode part of instruction */ 1045 req->ist.ei0 = vq_cmd_w0.u64; 1046 req->ist.ei3 = vq_cmd_w3.u64; 1047 1048 req->op = op; 1049 1050 *prep_req = req; 1051 return 0; 1052 } 1053 1054 static __rte_always_inline int 1055 cpt_dec_hmac_prep(uint32_t flags, 1056 uint64_t d_offs, 1057 uint64_t d_lens, 1058 fc_params_t *fc_params, 1059 void *op, 1060 void **prep_req) 1061 { 1062 uint32_t iv_offset = 0, size; 1063 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1064 struct cpt_ctx *cpt_ctx; 1065 int32_t hash_type, mac_len, m_size; 1066 uint8_t iv_len = 16; 1067 struct cpt_request_info *req; 1068 buf_ptr_t *meta_p, *aad_buf = NULL; 1069 uint32_t encr_offset, auth_offset; 1070 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1071 uint32_t passthrough_len = 0; 1072 void *m_vaddr, *offset_vaddr; 1073 uint64_t m_dma, offset_dma, ctx_dma; 1074 opcode_info_t opcode; 1075 vq_cmd_word0_t vq_cmd_w0; 1076 vq_cmd_word3_t vq_cmd_w3; 1077 void *c_vaddr; 1078 uint64_t c_dma; 1079 1080 meta_p = &fc_params->meta_buf; 1081 m_vaddr = meta_p->vaddr; 1082 m_dma = meta_p->dma_addr; 1083 m_size = meta_p->size; 1084 1085 encr_offset = ENCR_OFFSET(d_offs); 1086 auth_offset = AUTH_OFFSET(d_offs); 1087 encr_data_len = ENCR_DLEN(d_lens); 1088 auth_data_len = AUTH_DLEN(d_lens); 1089 1090 if (unlikely(flags & VALID_AAD_BUF)) { 1091 /* 1092 * We dont support both aad 1093 * and auth data separately 1094 */ 1095 auth_data_len = 0; 1096 auth_offset = 0; 1097 aad_len = fc_params->aad_buf.size; 1098 aad_buf = &fc_params->aad_buf; 1099 } 1100 1101 cpt_ctx = fc_params->ctx_buf.vaddr; 1102 hash_type = cpt_ctx->hash_type; 1103 mac_len = cpt_ctx->mac_len; 1104 1105 if (hash_type == GMAC_TYPE) 1106 encr_data_len = 0; 1107 1108 if (unlikely(!(flags & VALID_IV_BUF))) { 1109 iv_len = 0; 1110 iv_offset = ENCR_IV_OFFSET(d_offs); 1111 } 1112 1113 if (unlikely(flags & VALID_AAD_BUF)) { 1114 /* 1115 * When AAD is given, data above encr_offset is pass through 1116 * Since AAD is given as separate pointer and not as offset, 1117 * this is a special case as we need to fragment input data 1118 * into passthrough + encr_data and then insert AAD in between. 1119 */ 1120 if (hash_type != GMAC_TYPE) { 1121 passthrough_len = encr_offset; 1122 auth_offset = passthrough_len + iv_len; 1123 encr_offset = passthrough_len + aad_len + iv_len; 1124 auth_data_len = aad_len + encr_data_len; 1125 } else { 1126 passthrough_len = 16 + aad_len; 1127 auth_offset = passthrough_len + iv_len; 1128 auth_data_len = aad_len; 1129 } 1130 } else { 1131 encr_offset += iv_len; 1132 auth_offset += iv_len; 1133 } 1134 1135 /* 1136 * Save initial space that followed app data for completion code & 1137 * alternate completion code to fall in same cache line as app data 1138 */ 1139 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1140 m_dma += COMPLETION_CODE_SIZE; 1141 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1142 (uint8_t *)m_vaddr; 1143 c_vaddr = (uint8_t *)m_vaddr + size; 1144 c_dma = m_dma + size; 1145 size += sizeof(cpt_res_s_t); 1146 1147 m_vaddr = (uint8_t *)m_vaddr + size; 1148 m_dma += size; 1149 m_size -= size; 1150 1151 /* start cpt request info structure at 8 byte alignment */ 1152 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1153 (uint8_t *)m_vaddr; 1154 1155 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1156 1157 size += sizeof(struct cpt_request_info); 1158 m_vaddr = (uint8_t *)m_vaddr + size; 1159 m_dma += size; 1160 m_size -= size; 1161 1162 /* Decryption */ 1163 opcode.s.major = CPT_MAJOR_OP_FC; 1164 opcode.s.minor = 1; 1165 1166 enc_dlen = encr_offset + encr_data_len; 1167 auth_dlen = auth_offset + auth_data_len; 1168 1169 if (auth_dlen > enc_dlen) { 1170 inputlen = auth_dlen + mac_len; 1171 outputlen = auth_dlen; 1172 } else { 1173 inputlen = enc_dlen + mac_len; 1174 outputlen = enc_dlen; 1175 } 1176 1177 if (hash_type == GMAC_TYPE) 1178 encr_offset = inputlen; 1179 1180 vq_cmd_w0.u64 = 0; 1181 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len); 1182 vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len); 1183 1184 /* 1185 * In 83XX since we have a limitation of 1186 * IV & Offset control word not part of instruction 1187 * and need to be part of Data Buffer, we check if 1188 * head room is there and then only do the Direct mode processing 1189 */ 1190 if (likely((flags & SINGLE_BUF_INPLACE) && 1191 (flags & SINGLE_BUF_HEADTAILROOM))) { 1192 void *dm_vaddr = fc_params->bufs[0].vaddr; 1193 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1194 /* 1195 * This flag indicates that there is 24 bytes head room and 1196 * 8 bytes tail room available, so that we get to do 1197 * DIRECT MODE with limitation 1198 */ 1199 1200 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1201 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1202 req->ist.ei1 = offset_dma; 1203 1204 /* RPTR should just exclude offset control word */ 1205 req->ist.ei2 = dm_dma_addr - iv_len; 1206 1207 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1208 outputlen - iv_len); 1209 /* since this is decryption, 1210 * don't touch the content of 1211 * alternate ccode space as it contains 1212 * hmac. 1213 */ 1214 1215 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN); 1216 1217 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 1218 1219 if (likely(iv_len)) { 1220 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1221 OFF_CTRL_LEN); 1222 uint64_t *src = fc_params->iv_buf; 1223 dest[0] = src[0]; 1224 dest[1] = src[1]; 1225 } 1226 1227 *(uint64_t *)offset_vaddr = 1228 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1229 ((uint64_t)iv_offset << 8) | 1230 ((uint64_t)auth_offset)); 1231 1232 } else { 1233 uint64_t dptr_dma, rptr_dma; 1234 uint32_t g_size_bytes, s_size_bytes; 1235 sg_comp_t *gather_comp; 1236 sg_comp_t *scatter_comp; 1237 uint8_t *in_buffer; 1238 uint8_t i = 0; 1239 1240 /* This falls under strict SG mode */ 1241 offset_vaddr = m_vaddr; 1242 offset_dma = m_dma; 1243 size = OFF_CTRL_LEN + iv_len; 1244 1245 m_vaddr = (uint8_t *)m_vaddr + size; 1246 m_dma += size; 1247 m_size -= size; 1248 1249 opcode.s.major |= CPT_DMA_MODE; 1250 1251 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 1252 1253 if (likely(iv_len)) { 1254 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1255 OFF_CTRL_LEN); 1256 uint64_t *src = fc_params->iv_buf; 1257 dest[0] = src[0]; 1258 dest[1] = src[1]; 1259 } 1260 1261 *(uint64_t *)offset_vaddr = 1262 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1263 ((uint64_t)iv_offset << 8) | 1264 ((uint64_t)auth_offset)); 1265 1266 /* DPTR has SG list */ 1267 in_buffer = m_vaddr; 1268 dptr_dma = m_dma; 1269 1270 ((uint16_t *)in_buffer)[0] = 0; 1271 ((uint16_t *)in_buffer)[1] = 0; 1272 1273 /* TODO Add error check if space will be sufficient */ 1274 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1275 1276 /* 1277 * Input Gather List 1278 */ 1279 i = 0; 1280 1281 /* Offset control word that includes iv */ 1282 i = fill_sg_comp(gather_comp, i, offset_dma, 1283 OFF_CTRL_LEN + iv_len); 1284 1285 /* Add input data */ 1286 if (flags & VALID_MAC_BUF) { 1287 size = inputlen - iv_len - mac_len; 1288 if (size) { 1289 /* input data only */ 1290 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1291 i = fill_sg_comp_from_buf_min( 1292 gather_comp, i, 1293 fc_params->bufs, 1294 &size); 1295 } else { 1296 uint32_t aad_offset = aad_len ? 1297 passthrough_len : 0; 1298 1299 i = fill_sg_comp_from_iov(gather_comp, 1300 i, 1301 fc_params->src_iov, 1302 0, &size, 1303 aad_buf, 1304 aad_offset); 1305 } 1306 if (size) 1307 return ERR_BAD_INPUT_ARG; 1308 } 1309 1310 /* mac data */ 1311 if (mac_len) { 1312 i = fill_sg_comp_from_buf(gather_comp, i, 1313 &fc_params->mac_buf); 1314 } 1315 } else { 1316 /* input data + mac */ 1317 size = inputlen - iv_len; 1318 if (size) { 1319 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1320 i = fill_sg_comp_from_buf_min( 1321 gather_comp, i, 1322 fc_params->bufs, 1323 &size); 1324 } else { 1325 uint32_t aad_offset = aad_len ? 1326 passthrough_len : 0; 1327 1328 if (!fc_params->src_iov) 1329 return ERR_BAD_INPUT_ARG; 1330 1331 i = fill_sg_comp_from_iov( 1332 gather_comp, i, 1333 fc_params->src_iov, 1334 0, &size, 1335 aad_buf, 1336 aad_offset); 1337 } 1338 1339 if (size) 1340 return ERR_BAD_INPUT_ARG; 1341 } 1342 } 1343 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1344 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1345 1346 /* 1347 * Output Scatter List 1348 */ 1349 1350 i = 0; 1351 scatter_comp = 1352 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1353 1354 /* Add iv */ 1355 if (iv_len) { 1356 i = fill_sg_comp(scatter_comp, i, 1357 offset_dma + OFF_CTRL_LEN, 1358 iv_len); 1359 } 1360 1361 /* Add output data */ 1362 size = outputlen - iv_len; 1363 if (size) { 1364 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1365 /* handle single buffer here */ 1366 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1367 fc_params->bufs, 1368 &size); 1369 } else { 1370 uint32_t aad_offset = aad_len ? 1371 passthrough_len : 0; 1372 1373 if (!fc_params->dst_iov) 1374 return ERR_BAD_INPUT_ARG; 1375 1376 i = fill_sg_comp_from_iov(scatter_comp, i, 1377 fc_params->dst_iov, 0, 1378 &size, aad_buf, 1379 aad_offset); 1380 } 1381 1382 if (unlikely(size)) 1383 return ERR_BAD_INPUT_ARG; 1384 } 1385 1386 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1387 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1388 1389 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1390 1391 /* This is DPTR len incase of SG mode */ 1392 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 1393 1394 m_vaddr = (uint8_t *)m_vaddr + size; 1395 m_dma += size; 1396 m_size -= size; 1397 1398 /* cpt alternate completion address saved earlier */ 1399 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1400 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1401 rptr_dma = c_dma - 8; 1402 size += COMPLETION_CODE_SIZE; 1403 1404 req->ist.ei1 = dptr_dma; 1405 req->ist.ei2 = rptr_dma; 1406 } 1407 1408 /* First 16-bit swap then 64-bit swap */ 1409 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 1410 * to eliminate all the swapping 1411 */ 1412 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 1413 1414 ctx_dma = fc_params->ctx_buf.dma_addr + 1415 offsetof(struct cpt_ctx, fctx); 1416 /* vq command w3 */ 1417 vq_cmd_w3.u64 = 0; 1418 vq_cmd_w3.s.grp = 0; 1419 vq_cmd_w3.s.cptr = ctx_dma; 1420 1421 /* 16 byte aligned cpt res address */ 1422 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1423 *req->completion_addr = COMPLETION_CODE_INIT; 1424 req->comp_baddr = c_dma; 1425 1426 /* Fill microcode part of instruction */ 1427 req->ist.ei0 = vq_cmd_w0.u64; 1428 req->ist.ei3 = vq_cmd_w3.u64; 1429 1430 req->op = op; 1431 1432 *prep_req = req; 1433 return 0; 1434 } 1435 1436 static __rte_always_inline int 1437 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1438 uint64_t d_offs, 1439 uint64_t d_lens, 1440 fc_params_t *params, 1441 void *op, 1442 void **prep_req) 1443 { 1444 uint32_t size; 1445 int32_t inputlen, outputlen; 1446 struct cpt_ctx *cpt_ctx; 1447 uint32_t mac_len = 0; 1448 uint8_t snow3g, j; 1449 struct cpt_request_info *req; 1450 buf_ptr_t *buf_p; 1451 uint32_t encr_offset = 0, auth_offset = 0; 1452 uint32_t encr_data_len = 0, auth_data_len = 0; 1453 int flags, iv_len = 16, m_size; 1454 void *m_vaddr, *c_vaddr; 1455 uint64_t m_dma, c_dma, offset_ctrl; 1456 uint64_t *offset_vaddr, offset_dma; 1457 uint32_t *iv_s, iv[4]; 1458 vq_cmd_word0_t vq_cmd_w0; 1459 vq_cmd_word3_t vq_cmd_w3; 1460 opcode_info_t opcode; 1461 1462 buf_p = ¶ms->meta_buf; 1463 m_vaddr = buf_p->vaddr; 1464 m_dma = buf_p->dma_addr; 1465 m_size = buf_p->size; 1466 1467 cpt_ctx = params->ctx_buf.vaddr; 1468 flags = cpt_ctx->zsk_flags; 1469 mac_len = cpt_ctx->mac_len; 1470 snow3g = cpt_ctx->snow3g; 1471 1472 /* 1473 * Save initial space that followed app data for completion code & 1474 * alternate completion code to fall in same cache line as app data 1475 */ 1476 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1477 m_dma += COMPLETION_CODE_SIZE; 1478 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1479 (uint8_t *)m_vaddr; 1480 1481 c_vaddr = (uint8_t *)m_vaddr + size; 1482 c_dma = m_dma + size; 1483 size += sizeof(cpt_res_s_t); 1484 1485 m_vaddr = (uint8_t *)m_vaddr + size; 1486 m_dma += size; 1487 m_size -= size; 1488 1489 /* Reserve memory for cpt request info */ 1490 req = m_vaddr; 1491 1492 size = sizeof(struct cpt_request_info); 1493 m_vaddr = (uint8_t *)m_vaddr + size; 1494 m_dma += size; 1495 m_size -= size; 1496 1497 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1498 1499 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1500 opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) | 1501 (0 << 3) | (flags & 0x7)); 1502 1503 if (flags == 0x1) { 1504 /* 1505 * Microcode expects offsets in bytes 1506 * TODO: Rounding off 1507 */ 1508 auth_data_len = AUTH_DLEN(d_lens); 1509 1510 /* EIA3 or UIA2 */ 1511 auth_offset = AUTH_OFFSET(d_offs); 1512 auth_offset = auth_offset / 8; 1513 1514 /* consider iv len */ 1515 auth_offset += iv_len; 1516 1517 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1518 outputlen = mac_len; 1519 1520 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1521 1522 } else { 1523 /* EEA3 or UEA2 */ 1524 /* 1525 * Microcode expects offsets in bytes 1526 * TODO: Rounding off 1527 */ 1528 encr_data_len = ENCR_DLEN(d_lens); 1529 1530 encr_offset = ENCR_OFFSET(d_offs); 1531 encr_offset = encr_offset / 8; 1532 /* consider iv len */ 1533 encr_offset += iv_len; 1534 1535 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1536 outputlen = inputlen; 1537 1538 /* iv offset is 0 */ 1539 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1540 } 1541 1542 /* IV */ 1543 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1544 params->iv_buf; 1545 1546 if (snow3g) { 1547 /* 1548 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1549 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1550 */ 1551 1552 for (j = 0; j < 4; j++) 1553 iv[j] = iv_s[3 - j]; 1554 } else { 1555 /* ZUC doesn't need a swap */ 1556 for (j = 0; j < 4; j++) 1557 iv[j] = iv_s[j]; 1558 } 1559 1560 /* 1561 * GP op header, lengths are expected in bits. 1562 */ 1563 vq_cmd_w0.u64 = 0; 1564 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len); 1565 vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len); 1566 1567 /* 1568 * In 83XX since we have a limitation of 1569 * IV & Offset control word not part of instruction 1570 * and need to be part of Data Buffer, we check if 1571 * head room is there and then only do the Direct mode processing 1572 */ 1573 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1574 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1575 void *dm_vaddr = params->bufs[0].vaddr; 1576 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1577 /* 1578 * This flag indicates that there is 24 bytes head room and 1579 * 8 bytes tail room available, so that we get to do 1580 * DIRECT MODE with limitation 1581 */ 1582 1583 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1584 OFF_CTRL_LEN - iv_len); 1585 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1586 1587 /* DPTR */ 1588 req->ist.ei1 = offset_dma; 1589 /* RPTR should just exclude offset control word */ 1590 req->ist.ei2 = dm_dma_addr - iv_len; 1591 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1592 + outputlen - iv_len); 1593 1594 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN); 1595 1596 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 1597 1598 if (likely(iv_len)) { 1599 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1600 + OFF_CTRL_LEN); 1601 memcpy(iv_d, iv, 16); 1602 } 1603 1604 *offset_vaddr = offset_ctrl; 1605 } else { 1606 uint32_t i, g_size_bytes, s_size_bytes; 1607 uint64_t dptr_dma, rptr_dma; 1608 sg_comp_t *gather_comp; 1609 sg_comp_t *scatter_comp; 1610 uint8_t *in_buffer; 1611 uint32_t *iv_d; 1612 1613 /* save space for iv */ 1614 offset_vaddr = m_vaddr; 1615 offset_dma = m_dma; 1616 1617 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1618 m_dma += OFF_CTRL_LEN + iv_len; 1619 m_size -= OFF_CTRL_LEN + iv_len; 1620 1621 opcode.s.major |= CPT_DMA_MODE; 1622 1623 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 1624 1625 /* DPTR has SG list */ 1626 in_buffer = m_vaddr; 1627 dptr_dma = m_dma; 1628 1629 ((uint16_t *)in_buffer)[0] = 0; 1630 ((uint16_t *)in_buffer)[1] = 0; 1631 1632 /* TODO Add error check if space will be sufficient */ 1633 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1634 1635 /* 1636 * Input Gather List 1637 */ 1638 i = 0; 1639 1640 /* Offset control word followed by iv */ 1641 1642 i = fill_sg_comp(gather_comp, i, offset_dma, 1643 OFF_CTRL_LEN + iv_len); 1644 1645 /* iv offset is 0 */ 1646 *offset_vaddr = offset_ctrl; 1647 1648 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1649 memcpy(iv_d, iv, 16); 1650 1651 /* input data */ 1652 size = inputlen - iv_len; 1653 if (size) { 1654 i = fill_sg_comp_from_iov(gather_comp, i, 1655 params->src_iov, 1656 0, &size, NULL, 0); 1657 if (size) 1658 return ERR_BAD_INPUT_ARG; 1659 } 1660 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1661 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1662 1663 /* 1664 * Output Scatter List 1665 */ 1666 1667 i = 0; 1668 scatter_comp = 1669 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1670 1671 if (flags == 0x1) { 1672 /* IV in SLIST only for EEA3 & UEA2 */ 1673 iv_len = 0; 1674 } 1675 1676 if (iv_len) { 1677 i = fill_sg_comp(scatter_comp, i, 1678 offset_dma + OFF_CTRL_LEN, iv_len); 1679 } 1680 1681 /* Add output data */ 1682 if (req_flags & VALID_MAC_BUF) { 1683 size = outputlen - iv_len - mac_len; 1684 if (size) { 1685 i = fill_sg_comp_from_iov(scatter_comp, i, 1686 params->dst_iov, 0, 1687 &size, NULL, 0); 1688 1689 if (size) 1690 return ERR_BAD_INPUT_ARG; 1691 } 1692 1693 /* mac data */ 1694 if (mac_len) { 1695 i = fill_sg_comp_from_buf(scatter_comp, i, 1696 ¶ms->mac_buf); 1697 } 1698 } else { 1699 /* Output including mac */ 1700 size = outputlen - iv_len; 1701 if (size) { 1702 i = fill_sg_comp_from_iov(scatter_comp, i, 1703 params->dst_iov, 0, 1704 &size, NULL, 0); 1705 1706 if (size) 1707 return ERR_BAD_INPUT_ARG; 1708 } 1709 } 1710 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1711 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1712 1713 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1714 1715 /* This is DPTR len incase of SG mode */ 1716 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 1717 1718 m_vaddr = (uint8_t *)m_vaddr + size; 1719 m_dma += size; 1720 m_size -= size; 1721 1722 /* cpt alternate completion address saved earlier */ 1723 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1724 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1725 rptr_dma = c_dma - 8; 1726 1727 req->ist.ei1 = dptr_dma; 1728 req->ist.ei2 = rptr_dma; 1729 } 1730 1731 /* First 16-bit swap then 64-bit swap */ 1732 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 1733 * to eliminate all the swapping 1734 */ 1735 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 1736 1737 /* vq command w3 */ 1738 vq_cmd_w3.u64 = 0; 1739 vq_cmd_w3.s.grp = 0; 1740 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1741 offsetof(struct cpt_ctx, zs_ctx); 1742 1743 /* 16 byte aligned cpt res address */ 1744 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1745 *req->completion_addr = COMPLETION_CODE_INIT; 1746 req->comp_baddr = c_dma; 1747 1748 /* Fill microcode part of instruction */ 1749 req->ist.ei0 = vq_cmd_w0.u64; 1750 req->ist.ei3 = vq_cmd_w3.u64; 1751 1752 req->op = op; 1753 1754 *prep_req = req; 1755 return 0; 1756 } 1757 1758 static __rte_always_inline int 1759 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1760 uint64_t d_offs, 1761 uint64_t d_lens, 1762 fc_params_t *params, 1763 void *op, 1764 void **prep_req) 1765 { 1766 uint32_t size; 1767 int32_t inputlen = 0, outputlen; 1768 struct cpt_ctx *cpt_ctx; 1769 uint8_t snow3g, iv_len = 16; 1770 struct cpt_request_info *req; 1771 buf_ptr_t *buf_p; 1772 uint32_t encr_offset; 1773 uint32_t encr_data_len; 1774 int flags, m_size; 1775 void *m_vaddr, *c_vaddr; 1776 uint64_t m_dma, c_dma; 1777 uint64_t *offset_vaddr, offset_dma; 1778 uint32_t *iv_s, iv[4], j; 1779 vq_cmd_word0_t vq_cmd_w0; 1780 vq_cmd_word3_t vq_cmd_w3; 1781 opcode_info_t opcode; 1782 1783 buf_p = ¶ms->meta_buf; 1784 m_vaddr = buf_p->vaddr; 1785 m_dma = buf_p->dma_addr; 1786 m_size = buf_p->size; 1787 1788 /* 1789 * Microcode expects offsets in bytes 1790 * TODO: Rounding off 1791 */ 1792 encr_offset = ENCR_OFFSET(d_offs) / 8; 1793 encr_data_len = ENCR_DLEN(d_lens); 1794 1795 cpt_ctx = params->ctx_buf.vaddr; 1796 flags = cpt_ctx->zsk_flags; 1797 snow3g = cpt_ctx->snow3g; 1798 /* 1799 * Save initial space that followed app data for completion code & 1800 * alternate completion code to fall in same cache line as app data 1801 */ 1802 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1803 m_dma += COMPLETION_CODE_SIZE; 1804 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1805 (uint8_t *)m_vaddr; 1806 1807 c_vaddr = (uint8_t *)m_vaddr + size; 1808 c_dma = m_dma + size; 1809 size += sizeof(cpt_res_s_t); 1810 1811 m_vaddr = (uint8_t *)m_vaddr + size; 1812 m_dma += size; 1813 m_size -= size; 1814 1815 /* Reserve memory for cpt request info */ 1816 req = m_vaddr; 1817 1818 size = sizeof(struct cpt_request_info); 1819 m_vaddr = (uint8_t *)m_vaddr + size; 1820 m_dma += size; 1821 m_size -= size; 1822 1823 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1824 1825 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1826 opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) | 1827 (0 << 3) | (flags & 0x7)); 1828 1829 /* consider iv len */ 1830 encr_offset += iv_len; 1831 1832 inputlen = encr_offset + 1833 (RTE_ALIGN(encr_data_len, 8) / 8); 1834 outputlen = inputlen; 1835 1836 /* IV */ 1837 iv_s = params->iv_buf; 1838 if (snow3g) { 1839 /* 1840 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1841 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1842 */ 1843 1844 for (j = 0; j < 4; j++) 1845 iv[j] = iv_s[3 - j]; 1846 } else { 1847 /* ZUC doesn't need a swap */ 1848 for (j = 0; j < 4; j++) 1849 iv[j] = iv_s[j]; 1850 } 1851 1852 /* 1853 * GP op header, lengths are expected in bits. 1854 */ 1855 vq_cmd_w0.u64 = 0; 1856 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len); 1857 1858 /* 1859 * In 83XX since we have a limitation of 1860 * IV & Offset control word not part of instruction 1861 * and need to be part of Data Buffer, we check if 1862 * head room is there and then only do the Direct mode processing 1863 */ 1864 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1865 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1866 void *dm_vaddr = params->bufs[0].vaddr; 1867 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1868 /* 1869 * This flag indicates that there is 24 bytes head room and 1870 * 8 bytes tail room available, so that we get to do 1871 * DIRECT MODE with limitation 1872 */ 1873 1874 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1875 OFF_CTRL_LEN - iv_len); 1876 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1877 1878 /* DPTR */ 1879 req->ist.ei1 = offset_dma; 1880 /* RPTR should just exclude offset control word */ 1881 req->ist.ei2 = dm_dma_addr - iv_len; 1882 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1883 + outputlen - iv_len); 1884 1885 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN); 1886 1887 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 1888 1889 if (likely(iv_len)) { 1890 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1891 + OFF_CTRL_LEN); 1892 memcpy(iv_d, iv, 16); 1893 } 1894 1895 /* iv offset is 0 */ 1896 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1897 } else { 1898 uint32_t i, g_size_bytes, s_size_bytes; 1899 uint64_t dptr_dma, rptr_dma; 1900 sg_comp_t *gather_comp; 1901 sg_comp_t *scatter_comp; 1902 uint8_t *in_buffer; 1903 uint32_t *iv_d; 1904 1905 /* save space for offset and iv... */ 1906 offset_vaddr = m_vaddr; 1907 offset_dma = m_dma; 1908 1909 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1910 m_dma += OFF_CTRL_LEN + iv_len; 1911 m_size -= OFF_CTRL_LEN + iv_len; 1912 1913 opcode.s.major |= CPT_DMA_MODE; 1914 1915 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 1916 1917 /* DPTR has SG list */ 1918 in_buffer = m_vaddr; 1919 dptr_dma = m_dma; 1920 1921 ((uint16_t *)in_buffer)[0] = 0; 1922 ((uint16_t *)in_buffer)[1] = 0; 1923 1924 /* TODO Add error check if space will be sufficient */ 1925 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1926 1927 /* 1928 * Input Gather List 1929 */ 1930 i = 0; 1931 1932 /* Offset control word */ 1933 1934 /* iv offset is 0 */ 1935 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1936 1937 i = fill_sg_comp(gather_comp, i, offset_dma, 1938 OFF_CTRL_LEN + iv_len); 1939 1940 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1941 memcpy(iv_d, iv, 16); 1942 1943 /* Add input data */ 1944 size = inputlen - iv_len; 1945 if (size) { 1946 i = fill_sg_comp_from_iov(gather_comp, i, 1947 params->src_iov, 1948 0, &size, NULL, 0); 1949 if (size) 1950 return ERR_BAD_INPUT_ARG; 1951 } 1952 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1953 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1954 1955 /* 1956 * Output Scatter List 1957 */ 1958 1959 i = 0; 1960 scatter_comp = 1961 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1962 1963 /* IV */ 1964 i = fill_sg_comp(scatter_comp, i, 1965 offset_dma + OFF_CTRL_LEN, 1966 iv_len); 1967 1968 /* Add output data */ 1969 size = outputlen - iv_len; 1970 if (size) { 1971 i = fill_sg_comp_from_iov(scatter_comp, i, 1972 params->dst_iov, 0, 1973 &size, NULL, 0); 1974 1975 if (size) 1976 return ERR_BAD_INPUT_ARG; 1977 } 1978 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1979 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1980 1981 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1982 1983 /* This is DPTR len incase of SG mode */ 1984 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 1985 1986 m_vaddr = (uint8_t *)m_vaddr + size; 1987 m_dma += size; 1988 m_size -= size; 1989 1990 /* cpt alternate completion address saved earlier */ 1991 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1992 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1993 rptr_dma = c_dma - 8; 1994 1995 req->ist.ei1 = dptr_dma; 1996 req->ist.ei2 = rptr_dma; 1997 } 1998 1999 /* First 16-bit swap then 64-bit swap */ 2000 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 2001 * to eliminate all the swapping 2002 */ 2003 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 2004 2005 /* vq command w3 */ 2006 vq_cmd_w3.u64 = 0; 2007 vq_cmd_w3.s.grp = 0; 2008 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2009 offsetof(struct cpt_ctx, zs_ctx); 2010 2011 /* 16 byte aligned cpt res address */ 2012 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2013 *req->completion_addr = COMPLETION_CODE_INIT; 2014 req->comp_baddr = c_dma; 2015 2016 /* Fill microcode part of instruction */ 2017 req->ist.ei0 = vq_cmd_w0.u64; 2018 req->ist.ei3 = vq_cmd_w3.u64; 2019 2020 req->op = op; 2021 2022 *prep_req = req; 2023 return 0; 2024 } 2025 2026 static __rte_always_inline int 2027 cpt_kasumi_enc_prep(uint32_t req_flags, 2028 uint64_t d_offs, 2029 uint64_t d_lens, 2030 fc_params_t *params, 2031 void *op, 2032 void **prep_req) 2033 { 2034 uint32_t size; 2035 int32_t inputlen = 0, outputlen = 0; 2036 struct cpt_ctx *cpt_ctx; 2037 uint32_t mac_len = 0; 2038 uint8_t i = 0; 2039 struct cpt_request_info *req; 2040 buf_ptr_t *buf_p; 2041 uint32_t encr_offset, auth_offset; 2042 uint32_t encr_data_len, auth_data_len; 2043 int flags, m_size; 2044 uint8_t *iv_s, *iv_d, iv_len = 8; 2045 uint8_t dir = 0; 2046 void *m_vaddr, *c_vaddr; 2047 uint64_t m_dma, c_dma; 2048 uint64_t *offset_vaddr, offset_dma; 2049 vq_cmd_word0_t vq_cmd_w0; 2050 vq_cmd_word3_t vq_cmd_w3; 2051 opcode_info_t opcode; 2052 uint8_t *in_buffer; 2053 uint32_t g_size_bytes, s_size_bytes; 2054 uint64_t dptr_dma, rptr_dma; 2055 sg_comp_t *gather_comp; 2056 sg_comp_t *scatter_comp; 2057 2058 buf_p = ¶ms->meta_buf; 2059 m_vaddr = buf_p->vaddr; 2060 m_dma = buf_p->dma_addr; 2061 m_size = buf_p->size; 2062 2063 encr_offset = ENCR_OFFSET(d_offs) / 8; 2064 auth_offset = AUTH_OFFSET(d_offs) / 8; 2065 encr_data_len = ENCR_DLEN(d_lens); 2066 auth_data_len = AUTH_DLEN(d_lens); 2067 2068 cpt_ctx = params->ctx_buf.vaddr; 2069 flags = cpt_ctx->zsk_flags; 2070 mac_len = cpt_ctx->mac_len; 2071 2072 if (flags == 0x0) 2073 iv_s = params->iv_buf; 2074 else 2075 iv_s = params->auth_iv_buf; 2076 2077 dir = iv_s[8] & 0x1; 2078 2079 /* 2080 * Save initial space that followed app data for completion code & 2081 * alternate completion code to fall in same cache line as app data 2082 */ 2083 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2084 m_dma += COMPLETION_CODE_SIZE; 2085 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2086 (uint8_t *)m_vaddr; 2087 2088 c_vaddr = (uint8_t *)m_vaddr + size; 2089 c_dma = m_dma + size; 2090 size += sizeof(cpt_res_s_t); 2091 2092 m_vaddr = (uint8_t *)m_vaddr + size; 2093 m_dma += size; 2094 m_size -= size; 2095 2096 /* Reserve memory for cpt request info */ 2097 req = m_vaddr; 2098 2099 size = sizeof(struct cpt_request_info); 2100 m_vaddr = (uint8_t *)m_vaddr + size; 2101 m_dma += size; 2102 m_size -= size; 2103 2104 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2105 2106 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2107 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2108 (dir << 4) | (0 << 3) | (flags & 0x7)); 2109 2110 /* 2111 * GP op header, lengths are expected in bits. 2112 */ 2113 vq_cmd_w0.u64 = 0; 2114 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len); 2115 vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len); 2116 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 2117 2118 /* consider iv len */ 2119 if (flags == 0x0) { 2120 encr_offset += iv_len; 2121 auth_offset += iv_len; 2122 } 2123 2124 /* save space for offset ctrl and iv */ 2125 offset_vaddr = m_vaddr; 2126 offset_dma = m_dma; 2127 2128 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2129 m_dma += OFF_CTRL_LEN + iv_len; 2130 m_size -= OFF_CTRL_LEN + iv_len; 2131 2132 /* DPTR has SG list */ 2133 in_buffer = m_vaddr; 2134 dptr_dma = m_dma; 2135 2136 ((uint16_t *)in_buffer)[0] = 0; 2137 ((uint16_t *)in_buffer)[1] = 0; 2138 2139 /* TODO Add error check if space will be sufficient */ 2140 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2141 2142 /* 2143 * Input Gather List 2144 */ 2145 i = 0; 2146 2147 /* Offset control word followed by iv */ 2148 2149 if (flags == 0x0) { 2150 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2151 outputlen = inputlen; 2152 /* iv offset is 0 */ 2153 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2154 } else { 2155 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2156 outputlen = mac_len; 2157 /* iv offset is 0 */ 2158 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2159 } 2160 2161 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2162 2163 /* IV */ 2164 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2165 memcpy(iv_d, iv_s, iv_len); 2166 2167 /* input data */ 2168 size = inputlen - iv_len; 2169 if (size) { 2170 i = fill_sg_comp_from_iov(gather_comp, i, 2171 params->src_iov, 0, 2172 &size, NULL, 0); 2173 2174 if (size) 2175 return ERR_BAD_INPUT_ARG; 2176 } 2177 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2178 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2179 2180 /* 2181 * Output Scatter List 2182 */ 2183 2184 i = 0; 2185 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2186 2187 if (flags == 0x1) { 2188 /* IV in SLIST only for F8 */ 2189 iv_len = 0; 2190 } 2191 2192 /* IV */ 2193 if (iv_len) { 2194 i = fill_sg_comp(scatter_comp, i, 2195 offset_dma + OFF_CTRL_LEN, 2196 iv_len); 2197 } 2198 2199 /* Add output data */ 2200 if (req_flags & VALID_MAC_BUF) { 2201 size = outputlen - iv_len - mac_len; 2202 if (size) { 2203 i = fill_sg_comp_from_iov(scatter_comp, i, 2204 params->dst_iov, 0, 2205 &size, NULL, 0); 2206 2207 if (size) 2208 return ERR_BAD_INPUT_ARG; 2209 } 2210 2211 /* mac data */ 2212 if (mac_len) { 2213 i = fill_sg_comp_from_buf(scatter_comp, i, 2214 ¶ms->mac_buf); 2215 } 2216 } else { 2217 /* Output including mac */ 2218 size = outputlen - iv_len; 2219 if (size) { 2220 i = fill_sg_comp_from_iov(scatter_comp, i, 2221 params->dst_iov, 0, 2222 &size, NULL, 0); 2223 2224 if (size) 2225 return ERR_BAD_INPUT_ARG; 2226 } 2227 } 2228 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2229 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2230 2231 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2232 2233 /* This is DPTR len incase of SG mode */ 2234 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 2235 2236 m_vaddr = (uint8_t *)m_vaddr + size; 2237 m_dma += size; 2238 m_size -= size; 2239 2240 /* cpt alternate completion address saved earlier */ 2241 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2242 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2243 rptr_dma = c_dma - 8; 2244 2245 req->ist.ei1 = dptr_dma; 2246 req->ist.ei2 = rptr_dma; 2247 2248 /* First 16-bit swap then 64-bit swap */ 2249 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 2250 * to eliminate all the swapping 2251 */ 2252 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 2253 2254 /* vq command w3 */ 2255 vq_cmd_w3.u64 = 0; 2256 vq_cmd_w3.s.grp = 0; 2257 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2258 offsetof(struct cpt_ctx, k_ctx); 2259 2260 /* 16 byte aligned cpt res address */ 2261 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2262 *req->completion_addr = COMPLETION_CODE_INIT; 2263 req->comp_baddr = c_dma; 2264 2265 /* Fill microcode part of instruction */ 2266 req->ist.ei0 = vq_cmd_w0.u64; 2267 req->ist.ei3 = vq_cmd_w3.u64; 2268 2269 req->op = op; 2270 2271 *prep_req = req; 2272 return 0; 2273 } 2274 2275 static __rte_always_inline int 2276 cpt_kasumi_dec_prep(uint64_t d_offs, 2277 uint64_t d_lens, 2278 fc_params_t *params, 2279 void *op, 2280 void **prep_req) 2281 { 2282 uint32_t size; 2283 int32_t inputlen = 0, outputlen; 2284 struct cpt_ctx *cpt_ctx; 2285 uint8_t i = 0, iv_len = 8; 2286 struct cpt_request_info *req; 2287 buf_ptr_t *buf_p; 2288 uint32_t encr_offset; 2289 uint32_t encr_data_len; 2290 int flags, m_size; 2291 uint8_t dir = 0; 2292 void *m_vaddr, *c_vaddr; 2293 uint64_t m_dma, c_dma; 2294 uint64_t *offset_vaddr, offset_dma; 2295 vq_cmd_word0_t vq_cmd_w0; 2296 vq_cmd_word3_t vq_cmd_w3; 2297 opcode_info_t opcode; 2298 uint8_t *in_buffer; 2299 uint32_t g_size_bytes, s_size_bytes; 2300 uint64_t dptr_dma, rptr_dma; 2301 sg_comp_t *gather_comp; 2302 sg_comp_t *scatter_comp; 2303 2304 buf_p = ¶ms->meta_buf; 2305 m_vaddr = buf_p->vaddr; 2306 m_dma = buf_p->dma_addr; 2307 m_size = buf_p->size; 2308 2309 encr_offset = ENCR_OFFSET(d_offs) / 8; 2310 encr_data_len = ENCR_DLEN(d_lens); 2311 2312 cpt_ctx = params->ctx_buf.vaddr; 2313 flags = cpt_ctx->zsk_flags; 2314 /* 2315 * Save initial space that followed app data for completion code & 2316 * alternate completion code to fall in same cache line as app data 2317 */ 2318 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2319 m_dma += COMPLETION_CODE_SIZE; 2320 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2321 (uint8_t *)m_vaddr; 2322 2323 c_vaddr = (uint8_t *)m_vaddr + size; 2324 c_dma = m_dma + size; 2325 size += sizeof(cpt_res_s_t); 2326 2327 m_vaddr = (uint8_t *)m_vaddr + size; 2328 m_dma += size; 2329 m_size -= size; 2330 2331 /* Reserve memory for cpt request info */ 2332 req = m_vaddr; 2333 2334 size = sizeof(struct cpt_request_info); 2335 m_vaddr = (uint8_t *)m_vaddr + size; 2336 m_dma += size; 2337 m_size -= size; 2338 2339 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2340 2341 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2342 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2343 (dir << 4) | (0 << 3) | (flags & 0x7)); 2344 2345 /* 2346 * GP op header, lengths are expected in bits. 2347 */ 2348 vq_cmd_w0.u64 = 0; 2349 vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len); 2350 vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags); 2351 2352 /* consider iv len */ 2353 encr_offset += iv_len; 2354 2355 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2356 outputlen = inputlen; 2357 2358 /* save space for offset ctrl & iv */ 2359 offset_vaddr = m_vaddr; 2360 offset_dma = m_dma; 2361 2362 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2363 m_dma += OFF_CTRL_LEN + iv_len; 2364 m_size -= OFF_CTRL_LEN + iv_len; 2365 2366 /* DPTR has SG list */ 2367 in_buffer = m_vaddr; 2368 dptr_dma = m_dma; 2369 2370 ((uint16_t *)in_buffer)[0] = 0; 2371 ((uint16_t *)in_buffer)[1] = 0; 2372 2373 /* TODO Add error check if space will be sufficient */ 2374 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2375 2376 /* 2377 * Input Gather List 2378 */ 2379 i = 0; 2380 2381 /* Offset control word followed by iv */ 2382 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2383 2384 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2385 2386 /* IV */ 2387 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2388 params->iv_buf, iv_len); 2389 2390 /* Add input data */ 2391 size = inputlen - iv_len; 2392 if (size) { 2393 i = fill_sg_comp_from_iov(gather_comp, i, 2394 params->src_iov, 2395 0, &size, NULL, 0); 2396 if (size) 2397 return ERR_BAD_INPUT_ARG; 2398 } 2399 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2400 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2401 2402 /* 2403 * Output Scatter List 2404 */ 2405 2406 i = 0; 2407 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2408 2409 /* IV */ 2410 i = fill_sg_comp(scatter_comp, i, 2411 offset_dma + OFF_CTRL_LEN, 2412 iv_len); 2413 2414 /* Add output data */ 2415 size = outputlen - iv_len; 2416 if (size) { 2417 i = fill_sg_comp_from_iov(scatter_comp, i, 2418 params->dst_iov, 0, 2419 &size, NULL, 0); 2420 if (size) 2421 return ERR_BAD_INPUT_ARG; 2422 } 2423 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2424 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2425 2426 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2427 2428 /* This is DPTR len incase of SG mode */ 2429 vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size); 2430 2431 m_vaddr = (uint8_t *)m_vaddr + size; 2432 m_dma += size; 2433 m_size -= size; 2434 2435 /* cpt alternate completion address saved earlier */ 2436 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2437 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2438 rptr_dma = c_dma - 8; 2439 2440 req->ist.ei1 = dptr_dma; 2441 req->ist.ei2 = rptr_dma; 2442 2443 /* First 16-bit swap then 64-bit swap */ 2444 /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions 2445 * to eliminate all the swapping 2446 */ 2447 vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64); 2448 2449 /* vq command w3 */ 2450 vq_cmd_w3.u64 = 0; 2451 vq_cmd_w3.s.grp = 0; 2452 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2453 offsetof(struct cpt_ctx, k_ctx); 2454 2455 /* 16 byte aligned cpt res address */ 2456 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2457 *req->completion_addr = COMPLETION_CODE_INIT; 2458 req->comp_baddr = c_dma; 2459 2460 /* Fill microcode part of instruction */ 2461 req->ist.ei0 = vq_cmd_w0.u64; 2462 req->ist.ei3 = vq_cmd_w3.u64; 2463 2464 req->op = op; 2465 2466 *prep_req = req; 2467 return 0; 2468 } 2469 2470 static __rte_always_inline void * 2471 cpt_fc_dec_hmac_prep(uint32_t flags, 2472 uint64_t d_offs, 2473 uint64_t d_lens, 2474 fc_params_t *fc_params, 2475 void *op, int *ret_val) 2476 { 2477 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2478 uint8_t fc_type; 2479 void *prep_req = NULL; 2480 int ret; 2481 2482 fc_type = ctx->fc_type; 2483 2484 if (likely(fc_type == FC_GEN)) { 2485 ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, 2486 fc_params, op, &prep_req); 2487 } else if (fc_type == ZUC_SNOW3G) { 2488 ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, 2489 fc_params, op, &prep_req); 2490 } else if (fc_type == KASUMI) { 2491 ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, 2492 &prep_req); 2493 } else { 2494 /* 2495 * For AUTH_ONLY case, 2496 * MC only supports digest generation and verification 2497 * should be done in software by memcmp() 2498 */ 2499 2500 ret = ERR_EIO; 2501 } 2502 2503 if (unlikely(!prep_req)) 2504 *ret_val = ret; 2505 return prep_req; 2506 } 2507 2508 static __rte_always_inline void *__hot 2509 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2510 fc_params_t *fc_params, void *op, int *ret_val) 2511 { 2512 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2513 uint8_t fc_type; 2514 void *prep_req = NULL; 2515 int ret; 2516 2517 fc_type = ctx->fc_type; 2518 2519 /* Common api for rest of the ops */ 2520 if (likely(fc_type == FC_GEN)) { 2521 ret = cpt_enc_hmac_prep(flags, d_offs, d_lens, 2522 fc_params, op, &prep_req); 2523 } else if (fc_type == ZUC_SNOW3G) { 2524 ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, 2525 fc_params, op, &prep_req); 2526 } else if (fc_type == KASUMI) { 2527 ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens, 2528 fc_params, op, &prep_req); 2529 } else if (fc_type == HASH_HMAC) { 2530 ret = cpt_digest_gen_prep(flags, d_lens, fc_params, op, 2531 &prep_req); 2532 } else { 2533 ret = ERR_EIO; 2534 } 2535 2536 if (unlikely(!prep_req)) 2537 *ret_val = ret; 2538 return prep_req; 2539 } 2540 2541 static __rte_always_inline int 2542 cpt_fc_auth_set_key(void *ctx, auth_type_t type, uint8_t *key, 2543 uint16_t key_len, uint16_t mac_len) 2544 { 2545 struct cpt_ctx *cpt_ctx = ctx; 2546 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2547 uint64_t *ctrl_flags = NULL; 2548 2549 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2550 uint32_t keyx[4]; 2551 2552 if (key_len != 16) 2553 return -1; 2554 /* No support for AEAD yet */ 2555 if (cpt_ctx->enc_cipher) 2556 return -1; 2557 /* For ZUC/SNOW3G/Kasumi */ 2558 switch (type) { 2559 case SNOW3G_UIA2: 2560 cpt_ctx->snow3g = 1; 2561 gen_key_snow3g(key, keyx); 2562 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2563 cpt_ctx->fc_type = ZUC_SNOW3G; 2564 cpt_ctx->zsk_flags = 0x1; 2565 break; 2566 case ZUC_EIA3: 2567 cpt_ctx->snow3g = 0; 2568 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2569 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2570 cpt_ctx->fc_type = ZUC_SNOW3G; 2571 cpt_ctx->zsk_flags = 0x1; 2572 break; 2573 case KASUMI_F9_ECB: 2574 /* Kasumi ECB mode */ 2575 cpt_ctx->k_ecb = 1; 2576 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2577 cpt_ctx->fc_type = KASUMI; 2578 cpt_ctx->zsk_flags = 0x1; 2579 break; 2580 case KASUMI_F9_CBC: 2581 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2582 cpt_ctx->fc_type = KASUMI; 2583 cpt_ctx->zsk_flags = 0x1; 2584 break; 2585 default: 2586 return -1; 2587 } 2588 cpt_ctx->mac_len = 4; 2589 cpt_ctx->hash_type = type; 2590 return 0; 2591 } 2592 2593 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2594 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2595 cpt_ctx->fc_type = HASH_HMAC; 2596 } 2597 2598 ctrl_flags = (uint64_t *)&fctx->enc.enc_ctrl.flags; 2599 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 2600 2601 /* For GMAC auth, cipher must be NULL */ 2602 if (type == GMAC_TYPE) 2603 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 2604 2605 CPT_P_ENC_CTRL(fctx).hash_type = cpt_ctx->hash_type = type; 2606 CPT_P_ENC_CTRL(fctx).mac_len = cpt_ctx->mac_len = mac_len; 2607 2608 if (key_len) { 2609 cpt_ctx->hmac = 1; 2610 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2611 memcpy(cpt_ctx->auth_key, key, key_len); 2612 cpt_ctx->auth_key_len = key_len; 2613 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2614 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2615 memcpy(fctx->hmac.opad, key, key_len); 2616 CPT_P_ENC_CTRL(fctx).auth_input_type = 1; 2617 } 2618 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 2619 return 0; 2620 } 2621 2622 static __rte_always_inline int 2623 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2624 struct cpt_sess_misc *sess) 2625 { 2626 struct rte_crypto_aead_xform *aead_form; 2627 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2628 auth_type_t auth_type = 0; /* NULL Auth type */ 2629 uint32_t cipher_key_len = 0; 2630 uint8_t zsk_flag = 0, aes_gcm = 0; 2631 aead_form = &xform->aead; 2632 void *ctx; 2633 2634 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2635 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2636 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2637 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2638 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2639 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2640 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2641 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2642 } else { 2643 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2644 return -1; 2645 } 2646 switch (aead_form->algo) { 2647 case RTE_CRYPTO_AEAD_AES_GCM: 2648 enc_type = AES_GCM; 2649 cipher_key_len = 16; 2650 aes_gcm = 1; 2651 break; 2652 case RTE_CRYPTO_AEAD_AES_CCM: 2653 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2654 aead_form->algo); 2655 return -1; 2656 default: 2657 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2658 aead_form->algo); 2659 return -1; 2660 } 2661 if (aead_form->key.length < cipher_key_len) { 2662 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2663 (unsigned int long)aead_form->key.length); 2664 return -1; 2665 } 2666 sess->zsk_flag = zsk_flag; 2667 sess->aes_gcm = aes_gcm; 2668 sess->mac_len = aead_form->digest_length; 2669 sess->iv_offset = aead_form->iv.offset; 2670 sess->iv_length = aead_form->iv.length; 2671 sess->aad_length = aead_form->aad_length; 2672 ctx = (void *)((uint8_t *)sess + sizeof(struct cpt_sess_misc)), 2673 2674 cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2675 aead_form->key.length, NULL); 2676 2677 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length); 2678 2679 return 0; 2680 } 2681 2682 static __rte_always_inline int 2683 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2684 struct cpt_sess_misc *sess) 2685 { 2686 struct rte_crypto_cipher_xform *c_form; 2687 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2688 uint32_t cipher_key_len = 0; 2689 uint8_t zsk_flag = 0, aes_gcm = 0, aes_ctr = 0, is_null = 0; 2690 2691 if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) 2692 return -1; 2693 2694 c_form = &xform->cipher; 2695 2696 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2697 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2698 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2699 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2700 else { 2701 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2702 return -1; 2703 } 2704 2705 switch (c_form->algo) { 2706 case RTE_CRYPTO_CIPHER_AES_CBC: 2707 enc_type = AES_CBC; 2708 cipher_key_len = 16; 2709 break; 2710 case RTE_CRYPTO_CIPHER_3DES_CBC: 2711 enc_type = DES3_CBC; 2712 cipher_key_len = 24; 2713 break; 2714 case RTE_CRYPTO_CIPHER_DES_CBC: 2715 /* DES is implemented using 3DES in hardware */ 2716 enc_type = DES3_CBC; 2717 cipher_key_len = 8; 2718 break; 2719 case RTE_CRYPTO_CIPHER_AES_CTR: 2720 enc_type = AES_CTR; 2721 cipher_key_len = 16; 2722 aes_ctr = 1; 2723 break; 2724 case RTE_CRYPTO_CIPHER_NULL: 2725 enc_type = 0; 2726 is_null = 1; 2727 break; 2728 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2729 enc_type = KASUMI_F8_ECB; 2730 cipher_key_len = 16; 2731 zsk_flag = K_F8; 2732 break; 2733 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2734 enc_type = SNOW3G_UEA2; 2735 cipher_key_len = 16; 2736 zsk_flag = ZS_EA; 2737 break; 2738 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2739 enc_type = ZUC_EEA3; 2740 cipher_key_len = 16; 2741 zsk_flag = ZS_EA; 2742 break; 2743 case RTE_CRYPTO_CIPHER_AES_XTS: 2744 enc_type = AES_XTS; 2745 cipher_key_len = 16; 2746 break; 2747 case RTE_CRYPTO_CIPHER_3DES_ECB: 2748 enc_type = DES3_ECB; 2749 cipher_key_len = 24; 2750 break; 2751 case RTE_CRYPTO_CIPHER_AES_ECB: 2752 enc_type = AES_ECB; 2753 cipher_key_len = 16; 2754 break; 2755 case RTE_CRYPTO_CIPHER_3DES_CTR: 2756 case RTE_CRYPTO_CIPHER_AES_F8: 2757 case RTE_CRYPTO_CIPHER_ARC4: 2758 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2759 c_form->algo); 2760 return -1; 2761 default: 2762 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2763 c_form->algo); 2764 return -1; 2765 } 2766 2767 if (c_form->key.length < cipher_key_len) { 2768 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2769 (unsigned long) c_form->key.length); 2770 return -1; 2771 } 2772 2773 sess->zsk_flag = zsk_flag; 2774 sess->aes_gcm = aes_gcm; 2775 sess->aes_ctr = aes_ctr; 2776 sess->iv_offset = c_form->iv.offset; 2777 sess->iv_length = c_form->iv.length; 2778 sess->is_null = is_null; 2779 2780 cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data, 2781 c_form->key.length, NULL); 2782 2783 return 0; 2784 } 2785 2786 static __rte_always_inline int 2787 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2788 struct cpt_sess_misc *sess) 2789 { 2790 struct rte_crypto_auth_xform *a_form; 2791 auth_type_t auth_type = 0; /* NULL Auth type */ 2792 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2793 2794 if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) 2795 goto error_out; 2796 2797 a_form = &xform->auth; 2798 2799 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2800 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2801 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2802 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2803 else { 2804 CPT_LOG_DP_ERR("Unknown auth operation"); 2805 return -1; 2806 } 2807 2808 if (a_form->key.length > 64) { 2809 CPT_LOG_DP_ERR("Auth key length is big"); 2810 return -1; 2811 } 2812 2813 switch (a_form->algo) { 2814 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2815 /* Fall through */ 2816 case RTE_CRYPTO_AUTH_SHA1: 2817 auth_type = SHA1_TYPE; 2818 break; 2819 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2820 case RTE_CRYPTO_AUTH_SHA256: 2821 auth_type = SHA2_SHA256; 2822 break; 2823 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2824 case RTE_CRYPTO_AUTH_SHA512: 2825 auth_type = SHA2_SHA512; 2826 break; 2827 case RTE_CRYPTO_AUTH_AES_GMAC: 2828 auth_type = GMAC_TYPE; 2829 aes_gcm = 1; 2830 break; 2831 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2832 case RTE_CRYPTO_AUTH_SHA224: 2833 auth_type = SHA2_SHA224; 2834 break; 2835 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2836 case RTE_CRYPTO_AUTH_SHA384: 2837 auth_type = SHA2_SHA384; 2838 break; 2839 case RTE_CRYPTO_AUTH_MD5_HMAC: 2840 case RTE_CRYPTO_AUTH_MD5: 2841 auth_type = MD5_TYPE; 2842 break; 2843 case RTE_CRYPTO_AUTH_KASUMI_F9: 2844 auth_type = KASUMI_F9_ECB; 2845 /* 2846 * Indicate that direction needs to be taken out 2847 * from end of src 2848 */ 2849 zsk_flag = K_F9; 2850 break; 2851 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2852 auth_type = SNOW3G_UIA2; 2853 zsk_flag = ZS_IA; 2854 break; 2855 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2856 auth_type = ZUC_EIA3; 2857 zsk_flag = ZS_IA; 2858 break; 2859 case RTE_CRYPTO_AUTH_NULL: 2860 auth_type = 0; 2861 is_null = 1; 2862 break; 2863 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2864 case RTE_CRYPTO_AUTH_AES_CMAC: 2865 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2866 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2867 a_form->algo); 2868 goto error_out; 2869 default: 2870 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2871 a_form->algo); 2872 goto error_out; 2873 } 2874 2875 sess->zsk_flag = zsk_flag; 2876 sess->aes_gcm = aes_gcm; 2877 sess->mac_len = a_form->digest_length; 2878 sess->is_null = is_null; 2879 if (zsk_flag) { 2880 sess->auth_iv_offset = a_form->iv.offset; 2881 sess->auth_iv_length = a_form->iv.length; 2882 } 2883 cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data, 2884 a_form->key.length, a_form->digest_length); 2885 2886 return 0; 2887 2888 error_out: 2889 return -1; 2890 } 2891 2892 static __rte_always_inline int 2893 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2894 struct cpt_sess_misc *sess) 2895 { 2896 struct rte_crypto_auth_xform *a_form; 2897 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2898 auth_type_t auth_type = 0; /* NULL Auth type */ 2899 uint8_t zsk_flag = 0, aes_gcm = 0; 2900 void *ctx; 2901 2902 if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) 2903 return -1; 2904 2905 a_form = &xform->auth; 2906 2907 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2908 sess->cpt_op |= CPT_OP_ENCODE; 2909 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2910 sess->cpt_op |= CPT_OP_DECODE; 2911 else { 2912 CPT_LOG_DP_ERR("Unknown auth operation"); 2913 return -1; 2914 } 2915 2916 switch (a_form->algo) { 2917 case RTE_CRYPTO_AUTH_AES_GMAC: 2918 enc_type = AES_GCM; 2919 auth_type = GMAC_TYPE; 2920 break; 2921 default: 2922 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2923 a_form->algo); 2924 return -1; 2925 } 2926 2927 sess->zsk_flag = zsk_flag; 2928 sess->aes_gcm = aes_gcm; 2929 sess->is_gmac = 1; 2930 sess->iv_offset = a_form->iv.offset; 2931 sess->iv_length = a_form->iv.length; 2932 sess->mac_len = a_form->digest_length; 2933 ctx = (void *)((uint8_t *)sess + sizeof(struct cpt_sess_misc)), 2934 2935 cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2936 a_form->key.length, NULL); 2937 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length); 2938 2939 return 0; 2940 } 2941 2942 static __rte_always_inline void * 2943 alloc_op_meta(struct rte_mbuf *m_src, 2944 buf_ptr_t *buf, 2945 int32_t len, 2946 struct rte_mempool *cpt_meta_pool) 2947 { 2948 uint8_t *mdata; 2949 2950 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2951 if (likely(m_src && (m_src->nb_segs == 1))) { 2952 int32_t tailroom; 2953 phys_addr_t mphys; 2954 2955 /* Check if tailroom is sufficient to hold meta data */ 2956 tailroom = rte_pktmbuf_tailroom(m_src); 2957 if (likely(tailroom > len + 8)) { 2958 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2959 mphys = m_src->buf_physaddr + m_src->buf_len; 2960 mdata -= len; 2961 mphys -= len; 2962 buf->vaddr = mdata; 2963 buf->dma_addr = mphys; 2964 buf->size = len; 2965 /* Indicate that this is a mbuf allocated mdata */ 2966 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2967 return mdata; 2968 } 2969 } 2970 #else 2971 RTE_SET_USED(m_src); 2972 #endif 2973 2974 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2975 return NULL; 2976 2977 buf->vaddr = mdata; 2978 buf->dma_addr = rte_mempool_virt2iova(mdata); 2979 buf->size = len; 2980 2981 return mdata; 2982 } 2983 2984 /** 2985 * cpt_free_metabuf - free metabuf to mempool. 2986 * @param instance: pointer to instance. 2987 * @param objp: pointer to the metabuf. 2988 */ 2989 static __rte_always_inline void 2990 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2991 { 2992 bool nofree = ((uintptr_t)mdata & 1ull); 2993 2994 if (likely(nofree)) 2995 return; 2996 rte_mempool_put(cpt_meta_pool, mdata); 2997 } 2998 2999 static __rte_always_inline uint32_t 3000 prepare_iov_from_pkt(struct rte_mbuf *pkt, 3001 iov_ptr_t *iovec, uint32_t start_offset) 3002 { 3003 uint16_t index = 0; 3004 void *seg_data = NULL; 3005 phys_addr_t seg_phys; 3006 int32_t seg_size = 0; 3007 3008 if (!pkt) { 3009 iovec->buf_cnt = 0; 3010 return 0; 3011 } 3012 3013 if (!start_offset) { 3014 seg_data = rte_pktmbuf_mtod(pkt, void *); 3015 seg_phys = rte_pktmbuf_mtophys(pkt); 3016 seg_size = pkt->data_len; 3017 } else { 3018 while (start_offset >= pkt->data_len) { 3019 start_offset -= pkt->data_len; 3020 pkt = pkt->next; 3021 } 3022 3023 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 3024 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 3025 seg_size = pkt->data_len - start_offset; 3026 if (!seg_size) 3027 return 1; 3028 } 3029 3030 /* first seg */ 3031 iovec->bufs[index].vaddr = seg_data; 3032 iovec->bufs[index].dma_addr = seg_phys; 3033 iovec->bufs[index].size = seg_size; 3034 index++; 3035 pkt = pkt->next; 3036 3037 while (unlikely(pkt != NULL)) { 3038 seg_data = rte_pktmbuf_mtod(pkt, void *); 3039 seg_phys = rte_pktmbuf_mtophys(pkt); 3040 seg_size = pkt->data_len; 3041 if (!seg_size) 3042 break; 3043 3044 iovec->bufs[index].vaddr = seg_data; 3045 iovec->bufs[index].dma_addr = seg_phys; 3046 iovec->bufs[index].size = seg_size; 3047 3048 index++; 3049 3050 pkt = pkt->next; 3051 } 3052 3053 iovec->buf_cnt = index; 3054 return 0; 3055 } 3056 3057 static __rte_always_inline uint32_t 3058 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 3059 fc_params_t *param, 3060 uint32_t *flags) 3061 { 3062 uint16_t index = 0; 3063 void *seg_data = NULL; 3064 phys_addr_t seg_phys; 3065 uint32_t seg_size = 0; 3066 iov_ptr_t *iovec; 3067 3068 seg_data = rte_pktmbuf_mtod(pkt, void *); 3069 seg_phys = rte_pktmbuf_mtophys(pkt); 3070 seg_size = pkt->data_len; 3071 3072 /* first seg */ 3073 if (likely(!pkt->next)) { 3074 uint32_t headroom, tailroom; 3075 3076 *flags |= SINGLE_BUF_INPLACE; 3077 headroom = rte_pktmbuf_headroom(pkt); 3078 tailroom = rte_pktmbuf_tailroom(pkt); 3079 if (likely((headroom >= 24) && 3080 (tailroom >= 8))) { 3081 /* In 83XX this is prerequivisit for Direct mode */ 3082 *flags |= SINGLE_BUF_HEADTAILROOM; 3083 } 3084 param->bufs[0].vaddr = seg_data; 3085 param->bufs[0].dma_addr = seg_phys; 3086 param->bufs[0].size = seg_size; 3087 return 0; 3088 } 3089 iovec = param->src_iov; 3090 iovec->bufs[index].vaddr = seg_data; 3091 iovec->bufs[index].dma_addr = seg_phys; 3092 iovec->bufs[index].size = seg_size; 3093 index++; 3094 pkt = pkt->next; 3095 3096 while (unlikely(pkt != NULL)) { 3097 seg_data = rte_pktmbuf_mtod(pkt, void *); 3098 seg_phys = rte_pktmbuf_mtophys(pkt); 3099 seg_size = pkt->data_len; 3100 3101 if (!seg_size) 3102 break; 3103 3104 iovec->bufs[index].vaddr = seg_data; 3105 iovec->bufs[index].dma_addr = seg_phys; 3106 iovec->bufs[index].size = seg_size; 3107 3108 index++; 3109 3110 pkt = pkt->next; 3111 } 3112 3113 iovec->buf_cnt = index; 3114 return 0; 3115 } 3116 3117 static __rte_always_inline void * 3118 fill_fc_params(struct rte_crypto_op *cop, 3119 struct cpt_sess_misc *sess_misc, 3120 void **mdata_ptr, 3121 int *op_ret) 3122 { 3123 uint32_t space = 0; 3124 struct rte_crypto_sym_op *sym_op = cop->sym; 3125 void *mdata; 3126 uintptr_t *op; 3127 uint32_t mc_hash_off; 3128 uint32_t flags = 0; 3129 uint64_t d_offs, d_lens; 3130 void *prep_req = NULL; 3131 struct rte_mbuf *m_src, *m_dst; 3132 uint8_t cpt_op = sess_misc->cpt_op; 3133 uint8_t zsk_flag = sess_misc->zsk_flag; 3134 uint8_t aes_gcm = sess_misc->aes_gcm; 3135 uint16_t mac_len = sess_misc->mac_len; 3136 #ifdef CPT_ALWAYS_USE_SG_MODE 3137 uint8_t inplace = 0; 3138 #else 3139 uint8_t inplace = 1; 3140 #endif 3141 fc_params_t fc_params; 3142 char src[SRC_IOV_SIZE]; 3143 char dst[SRC_IOV_SIZE]; 3144 uint32_t iv_buf[4]; 3145 struct cptvf_meta_info *cpt_m_info = 3146 (struct cptvf_meta_info *)(*mdata_ptr); 3147 3148 if (likely(sess_misc->iv_length)) { 3149 flags |= VALID_IV_BUF; 3150 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3151 uint8_t *, sess_misc->iv_offset); 3152 if (sess_misc->aes_ctr && 3153 unlikely(sess_misc->iv_length != 16)) { 3154 memcpy((uint8_t *)iv_buf, 3155 rte_crypto_op_ctod_offset(cop, 3156 uint8_t *, sess_misc->iv_offset), 12); 3157 iv_buf[3] = rte_cpu_to_be_32(0x1); 3158 fc_params.iv_buf = iv_buf; 3159 } 3160 } 3161 3162 if (zsk_flag) { 3163 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3164 uint8_t *, 3165 sess_misc->auth_iv_offset); 3166 if (zsk_flag == K_F9) { 3167 CPT_LOG_DP_ERR("Should not reach here for " 3168 "kasumi F9\n"); 3169 } 3170 if (zsk_flag != ZS_EA) 3171 inplace = 0; 3172 } 3173 m_src = sym_op->m_src; 3174 m_dst = sym_op->m_dst; 3175 3176 if (aes_gcm) { 3177 uint8_t *salt; 3178 uint8_t *aad_data; 3179 uint16_t aad_len; 3180 3181 d_offs = sym_op->aead.data.offset; 3182 d_lens = sym_op->aead.data.length; 3183 mc_hash_off = sym_op->aead.data.offset + 3184 sym_op->aead.data.length; 3185 3186 aad_data = sym_op->aead.aad.data; 3187 aad_len = sess_misc->aad_length; 3188 if (likely((aad_data + aad_len) == 3189 rte_pktmbuf_mtod_offset(m_src, 3190 uint8_t *, 3191 sym_op->aead.data.offset))) { 3192 d_offs = (d_offs - aad_len) | (d_offs << 16); 3193 d_lens = (d_lens + aad_len) | (d_lens << 32); 3194 } else { 3195 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3196 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3197 fc_params.aad_buf.size = aad_len; 3198 flags |= VALID_AAD_BUF; 3199 inplace = 0; 3200 d_offs = d_offs << 16; 3201 d_lens = d_lens << 32; 3202 } 3203 3204 salt = fc_params.iv_buf; 3205 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3206 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3207 sess_misc->salt = *(uint32_t *)salt; 3208 } 3209 fc_params.iv_buf = salt + 4; 3210 if (likely(mac_len)) { 3211 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3212 m_src; 3213 3214 if (!m) 3215 m = m_src; 3216 3217 /* hmac immediately following data is best case */ 3218 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3219 mc_hash_off != 3220 (uint8_t *)sym_op->aead.digest.data)) { 3221 flags |= VALID_MAC_BUF; 3222 fc_params.mac_buf.size = sess_misc->mac_len; 3223 fc_params.mac_buf.vaddr = 3224 sym_op->aead.digest.data; 3225 fc_params.mac_buf.dma_addr = 3226 sym_op->aead.digest.phys_addr; 3227 inplace = 0; 3228 } 3229 } 3230 } else { 3231 d_offs = sym_op->cipher.data.offset; 3232 d_lens = sym_op->cipher.data.length; 3233 mc_hash_off = sym_op->cipher.data.offset + 3234 sym_op->cipher.data.length; 3235 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3236 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3237 3238 if (mc_hash_off < (sym_op->auth.data.offset + 3239 sym_op->auth.data.length)){ 3240 mc_hash_off = (sym_op->auth.data.offset + 3241 sym_op->auth.data.length); 3242 } 3243 /* for gmac, salt should be updated like in gcm */ 3244 if (unlikely(sess_misc->is_gmac)) { 3245 uint8_t *salt; 3246 salt = fc_params.iv_buf; 3247 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3248 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3249 sess_misc->salt = *(uint32_t *)salt; 3250 } 3251 fc_params.iv_buf = salt + 4; 3252 } 3253 if (likely(mac_len)) { 3254 struct rte_mbuf *m; 3255 3256 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3257 if (!m) 3258 m = m_src; 3259 3260 /* hmac immediately following data is best case */ 3261 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3262 mc_hash_off != 3263 (uint8_t *)sym_op->auth.digest.data)) { 3264 flags |= VALID_MAC_BUF; 3265 fc_params.mac_buf.size = 3266 sess_misc->mac_len; 3267 fc_params.mac_buf.vaddr = 3268 sym_op->auth.digest.data; 3269 fc_params.mac_buf.dma_addr = 3270 sym_op->auth.digest.phys_addr; 3271 inplace = 0; 3272 } 3273 } 3274 } 3275 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3276 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3277 3278 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3279 inplace = 0; 3280 3281 if (likely(!m_dst && inplace)) { 3282 /* Case of single buffer without AAD buf or 3283 * separate mac buf in place and 3284 * not air crypto 3285 */ 3286 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3287 3288 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3289 &fc_params, 3290 &flags))) { 3291 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3292 *op_ret = -1; 3293 return NULL; 3294 } 3295 3296 } else { 3297 /* Out of place processing */ 3298 fc_params.src_iov = (void *)src; 3299 fc_params.dst_iov = (void *)dst; 3300 3301 /* Store SG I/O in the api for reuse */ 3302 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3303 CPT_LOG_DP_ERR("Prepare src iov failed"); 3304 *op_ret = -1; 3305 return NULL; 3306 } 3307 3308 if (unlikely(m_dst != NULL)) { 3309 uint32_t pkt_len; 3310 3311 /* Try to make room as much as src has */ 3312 m_dst = sym_op->m_dst; 3313 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3314 3315 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3316 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3317 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3318 CPT_LOG_DP_ERR("Not enough space in " 3319 "m_dst %p, need %u" 3320 " more", 3321 m_dst, pkt_len); 3322 return NULL; 3323 } 3324 } 3325 3326 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3327 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3328 "m_dst %p", m_dst); 3329 return NULL; 3330 } 3331 } else { 3332 fc_params.dst_iov = (void *)src; 3333 } 3334 } 3335 3336 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3337 mdata = alloc_op_meta(m_src, 3338 &fc_params.meta_buf, 3339 cpt_m_info->cptvf_op_sb_mlen, 3340 cpt_m_info->cptvf_meta_pool); 3341 else 3342 mdata = alloc_op_meta(NULL, 3343 &fc_params.meta_buf, 3344 cpt_m_info->cptvf_op_mlen, 3345 cpt_m_info->cptvf_meta_pool); 3346 3347 if (unlikely(mdata == NULL)) { 3348 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3349 return NULL; 3350 } 3351 3352 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3353 op[0] = (uintptr_t)mdata; 3354 op[1] = (uintptr_t)cop; 3355 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3356 space += 4 * sizeof(uint64_t); 3357 3358 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3359 fc_params.meta_buf.dma_addr += space; 3360 fc_params.meta_buf.size -= space; 3361 3362 /* Finally prepare the instruction */ 3363 if (cpt_op & CPT_OP_ENCODE) 3364 prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3365 &fc_params, op, op_ret); 3366 else 3367 prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3368 &fc_params, op, op_ret); 3369 3370 if (unlikely(!prep_req)) 3371 free_op_meta(mdata, cpt_m_info->cptvf_meta_pool); 3372 *mdata_ptr = mdata; 3373 return prep_req; 3374 } 3375 3376 static __rte_always_inline void 3377 compl_auth_verify(struct rte_crypto_op *op, 3378 uint8_t *gen_mac, 3379 uint64_t mac_len) 3380 { 3381 uint8_t *mac; 3382 struct rte_crypto_sym_op *sym_op = op->sym; 3383 3384 if (sym_op->auth.digest.data) 3385 mac = sym_op->auth.digest.data; 3386 else 3387 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3388 uint8_t *, 3389 sym_op->auth.data.length + 3390 sym_op->auth.data.offset); 3391 if (!mac) { 3392 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3393 return; 3394 } 3395 3396 if (memcmp(mac, gen_mac, mac_len)) 3397 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3398 else 3399 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3400 } 3401 3402 static __rte_always_inline int 3403 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3404 { 3405 struct rte_crypto_sym_xform *chain; 3406 3407 CPT_PMD_INIT_FUNC_TRACE(); 3408 3409 if (cpt_is_algo_supported(xform)) 3410 goto err; 3411 3412 chain = xform; 3413 while (chain) { 3414 switch (chain->type) { 3415 case RTE_CRYPTO_SYM_XFORM_AEAD: 3416 if (fill_sess_aead(chain, sess)) 3417 goto err; 3418 break; 3419 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3420 if (fill_sess_cipher(chain, sess)) 3421 goto err; 3422 break; 3423 case RTE_CRYPTO_SYM_XFORM_AUTH: 3424 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3425 if (fill_sess_gmac(chain, sess)) 3426 goto err; 3427 } else { 3428 if (fill_sess_auth(chain, sess)) 3429 goto err; 3430 } 3431 break; 3432 default: 3433 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3434 break; 3435 } 3436 chain = chain->next; 3437 } 3438 3439 return 0; 3440 3441 err: 3442 return -1; 3443 } 3444 3445 static __rte_always_inline void 3446 find_kasumif9_direction_and_length(uint8_t *src, 3447 uint32_t counter_num_bytes, 3448 uint32_t *addr_length_in_bits, 3449 uint8_t *addr_direction) 3450 { 3451 uint8_t found = 0; 3452 uint32_t pos; 3453 uint8_t last_byte; 3454 while (!found && counter_num_bytes > 0) { 3455 counter_num_bytes--; 3456 if (src[counter_num_bytes] == 0x00) 3457 continue; 3458 pos = rte_bsf32(src[counter_num_bytes]); 3459 if (pos == 7) { 3460 if (likely(counter_num_bytes > 0)) { 3461 last_byte = src[counter_num_bytes - 1]; 3462 *addr_direction = last_byte & 0x1; 3463 *addr_length_in_bits = counter_num_bytes * 8 3464 - 1; 3465 } 3466 } else { 3467 last_byte = src[counter_num_bytes]; 3468 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3469 *addr_length_in_bits = counter_num_bytes * 8 3470 + (8 - (pos + 2)); 3471 } 3472 found = 1; 3473 } 3474 } 3475 3476 /* 3477 * This handles all auth only except AES_GMAC 3478 */ 3479 static __rte_always_inline void * 3480 fill_digest_params(struct rte_crypto_op *cop, 3481 struct cpt_sess_misc *sess, 3482 void **mdata_ptr, 3483 int *op_ret) 3484 { 3485 uint32_t space = 0; 3486 struct rte_crypto_sym_op *sym_op = cop->sym; 3487 void *mdata; 3488 phys_addr_t mphys; 3489 uint64_t *op; 3490 uint32_t auth_range_off; 3491 uint32_t flags = 0; 3492 uint64_t d_offs = 0, d_lens; 3493 void *prep_req = NULL; 3494 struct rte_mbuf *m_src, *m_dst; 3495 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3496 uint8_t zsk_flag = sess->zsk_flag; 3497 uint16_t mac_len = sess->mac_len; 3498 fc_params_t params; 3499 char src[SRC_IOV_SIZE]; 3500 uint8_t iv_buf[16]; 3501 memset(¶ms, 0, sizeof(fc_params_t)); 3502 struct cptvf_meta_info *cpt_m_info = 3503 (struct cptvf_meta_info *)(*mdata_ptr); 3504 3505 m_src = sym_op->m_src; 3506 3507 /* For just digest lets force mempool alloc */ 3508 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, cpt_m_info->cptvf_op_mlen, 3509 cpt_m_info->cptvf_meta_pool); 3510 if (mdata == NULL) { 3511 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3512 *op_ret = -ENOMEM; 3513 return NULL; 3514 } 3515 3516 mphys = params.meta_buf.dma_addr; 3517 3518 op = mdata; 3519 op[0] = (uintptr_t)mdata; 3520 op[1] = (uintptr_t)cop; 3521 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3522 space += 4 * sizeof(uint64_t); 3523 3524 auth_range_off = sym_op->auth.data.offset; 3525 3526 flags = VALID_MAC_BUF; 3527 params.src_iov = (void *)src; 3528 if (unlikely(zsk_flag)) { 3529 /* 3530 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3531 * we will send pass through even for auth only case, 3532 * let MC handle it 3533 */ 3534 d_offs = auth_range_off; 3535 auth_range_off = 0; 3536 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3537 uint8_t *, sess->auth_iv_offset); 3538 if (zsk_flag == K_F9) { 3539 uint32_t length_in_bits, num_bytes; 3540 uint8_t *src, direction = 0; 3541 uint32_t counter_num_bytes; 3542 3543 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3544 uint8_t *), 8); 3545 /* 3546 * This is kasumi f9, take direction from 3547 * source buffer 3548 */ 3549 length_in_bits = cop->sym->auth.data.length; 3550 num_bytes = (length_in_bits >> 3); 3551 counter_num_bytes = num_bytes; 3552 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3553 find_kasumif9_direction_and_length(src, 3554 counter_num_bytes, 3555 &length_in_bits, 3556 &direction); 3557 length_in_bits -= 64; 3558 cop->sym->auth.data.offset += 64; 3559 d_offs = cop->sym->auth.data.offset; 3560 auth_range_off = d_offs / 8; 3561 cop->sym->auth.data.length = length_in_bits; 3562 3563 /* Store it at end of auth iv */ 3564 iv_buf[8] = direction; 3565 params.auth_iv_buf = iv_buf; 3566 } 3567 } 3568 3569 d_lens = sym_op->auth.data.length; 3570 3571 params.ctx_buf.vaddr = SESS_PRIV(sess); 3572 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3573 3574 if (auth_op == CPT_OP_AUTH_GENERATE) { 3575 if (sym_op->auth.digest.data) { 3576 /* 3577 * Digest to be generated 3578 * in separate buffer 3579 */ 3580 params.mac_buf.size = 3581 sess->mac_len; 3582 params.mac_buf.vaddr = 3583 sym_op->auth.digest.data; 3584 params.mac_buf.dma_addr = 3585 sym_op->auth.digest.phys_addr; 3586 } else { 3587 uint32_t off = sym_op->auth.data.offset + 3588 sym_op->auth.data.length; 3589 int32_t dlen, space; 3590 3591 m_dst = sym_op->m_dst ? 3592 sym_op->m_dst : sym_op->m_src; 3593 dlen = rte_pktmbuf_pkt_len(m_dst); 3594 3595 space = off + mac_len - dlen; 3596 if (space > 0) 3597 if (!rte_pktmbuf_append(m_dst, space)) { 3598 CPT_LOG_DP_ERR("Failed to extend " 3599 "mbuf by %uB", space); 3600 goto err; 3601 } 3602 3603 params.mac_buf.vaddr = 3604 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3605 params.mac_buf.dma_addr = 3606 rte_pktmbuf_mtophys_offset(m_dst, off); 3607 params.mac_buf.size = mac_len; 3608 } 3609 } else { 3610 /* Need space for storing generated mac */ 3611 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3612 params.mac_buf.dma_addr = mphys + space; 3613 params.mac_buf.size = mac_len; 3614 space += RTE_ALIGN_CEIL(mac_len, 8); 3615 op[2] = (uintptr_t)params.mac_buf.vaddr; 3616 op[3] = mac_len; 3617 } 3618 3619 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3620 params.meta_buf.dma_addr = mphys + space; 3621 params.meta_buf.size -= space; 3622 3623 /* Out of place processing */ 3624 params.src_iov = (void *)src; 3625 3626 /*Store SG I/O in the api for reuse */ 3627 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3628 CPT_LOG_DP_ERR("Prepare src iov failed"); 3629 *op_ret = -1; 3630 goto err; 3631 } 3632 3633 prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3634 ¶ms, op, op_ret); 3635 *mdata_ptr = mdata; 3636 return prep_req; 3637 err: 3638 if (unlikely(!prep_req)) 3639 free_op_meta(mdata, cpt_m_info->cptvf_meta_pool); 3640 return NULL; 3641 } 3642 3643 #endif /*_CPT_UCODE_H_ */ 3644