1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2022 Intel Corporation 3 */ 4 5 #include <rte_cryptodev.h> 6 #include <cryptodev_pmd.h> 7 #include "qat_sym_session.h" 8 #include "qat_sym.h" 9 #include "qat_asym.h" 10 #include "qat_crypto.h" 11 #include "qat_crypto_pmd_gens.h" 12 13 14 static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen3[] = { 15 QAT_SYM_CIPHER_CAP(3DES_CBC, 16 CAP_SET(block_size, 8), 17 CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), 18 QAT_SYM_CIPHER_CAP(DES_CBC, 19 CAP_SET(block_size, 8), 20 CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), 21 QAT_SYM_CIPHER_CAP(3DES_CTR, 22 CAP_SET(block_size, 8), 23 CAP_RNG(key_size, 16, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), 24 QAT_SYM_PLAIN_AUTH_CAP(SHA1, 25 CAP_SET(block_size, 64), 26 CAP_RNG(digest_size, 1, 20, 1)), 27 QAT_SYM_AUTH_CAP(SHA224, 28 CAP_SET(block_size, 64), 29 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1), 30 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 31 QAT_SYM_AUTH_CAP(SHA224_HMAC, 32 CAP_SET(block_size, 64), 33 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1), 34 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 35 QAT_SYM_AUTH_CAP(SHA1_HMAC, 36 CAP_SET(block_size, 64), 37 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1), 38 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 39 QAT_SYM_AUTH_CAP(MD5_HMAC, 40 CAP_SET(block_size, 64), 41 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 16, 1), 42 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 43 QAT_SYM_CIPHER_CAP(DES_DOCSISBPI, 44 CAP_SET(block_size, 8), 45 CAP_RNG(key_size, 8, 8, 0), CAP_RNG(iv_size, 8, 8, 0)), 46 QAT_SYM_PLAIN_AUTH_CAP(SHA3_224, 47 CAP_SET(block_size, 144), 48 CAP_RNG(digest_size, 28, 28, 0)), 49 QAT_SYM_CIPHER_CAP(SM4_ECB, 50 CAP_SET(block_size, 16), 51 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 0, 0, 0)) 52 }; 53 54 static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen3[] = { 55 QAT_SYM_AEAD_CAP(AES_GCM, 56 CAP_SET(block_size, 16), 57 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), 58 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)), 59 QAT_SYM_AEAD_CAP(AES_CCM, 60 CAP_SET(block_size, 16), 61 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2), 62 CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)), 63 QAT_SYM_AUTH_CAP(AES_GMAC, 64 CAP_SET(block_size, 16), 65 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), 66 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)), 67 QAT_SYM_AUTH_CAP(AES_CMAC, 68 CAP_SET(block_size, 16), 69 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4), 70 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 71 QAT_SYM_AUTH_CAP(SHA256, 72 CAP_SET(block_size, 64), 73 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1), 74 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 75 QAT_SYM_AUTH_CAP(SHA384, 76 CAP_SET(block_size, 128), 77 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1), 78 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 79 QAT_SYM_AUTH_CAP(SHA512, 80 CAP_SET(block_size, 128), 81 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1), 82 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 83 QAT_SYM_PLAIN_AUTH_CAP(SHA3_256, 84 CAP_SET(block_size, 136), 85 CAP_RNG(digest_size, 32, 32, 0)), 86 QAT_SYM_PLAIN_AUTH_CAP(SHA3_384, 87 CAP_SET(block_size, 104), 88 CAP_RNG(digest_size, 48, 48, 0)), 89 QAT_SYM_PLAIN_AUTH_CAP(SHA3_512, 90 CAP_SET(block_size, 72), 91 CAP_RNG(digest_size, 64, 64, 0)), 92 QAT_SYM_AUTH_CAP(SHA256_HMAC, 93 CAP_SET(block_size, 64), 94 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1), 95 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 96 QAT_SYM_AUTH_CAP(SHA384_HMAC, 97 CAP_SET(block_size, 128), 98 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1), 99 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 100 QAT_SYM_AUTH_CAP(SHA512_HMAC, 101 CAP_SET(block_size, 128), 102 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1), 103 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 104 QAT_SYM_AUTH_CAP(AES_XCBC_MAC, 105 CAP_SET(block_size, 16), 106 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0), 107 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 108 QAT_SYM_AUTH_CAP(SNOW3G_UIA2, 109 CAP_SET(block_size, 16), 110 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), 111 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)), 112 QAT_SYM_AUTH_CAP(KASUMI_F9, 113 CAP_SET(block_size, 8), 114 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), 115 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 116 QAT_SYM_AUTH_CAP(NULL, 117 CAP_SET(block_size, 1), 118 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size), 119 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 120 QAT_SYM_CIPHER_CAP(AES_CBC, 121 CAP_SET(block_size, 16), 122 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), 123 QAT_SYM_CIPHER_CAP(AES_CTR, 124 CAP_SET(block_size, 16), 125 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), 126 QAT_SYM_CIPHER_CAP(AES_XTS, 127 CAP_SET(block_size, 16), 128 CAP_RNG(key_size, 32, 64, 32), CAP_RNG(iv_size, 16, 16, 0)), 129 QAT_SYM_CIPHER_CAP(AES_DOCSISBPI, 130 CAP_SET(block_size, 16), 131 CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)), 132 QAT_SYM_CIPHER_CAP(SNOW3G_UEA2, 133 CAP_SET(block_size, 16), 134 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), 135 QAT_SYM_CIPHER_CAP(KASUMI_F8, 136 CAP_SET(block_size, 8), 137 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 8, 8, 0)), 138 QAT_SYM_CIPHER_CAP(NULL, 139 CAP_SET(block_size, 1), 140 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)), 141 QAT_SYM_CIPHER_CAP(ZUC_EEA3, 142 CAP_SET(block_size, 16), 143 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), 144 QAT_SYM_AUTH_CAP(ZUC_EIA3, 145 CAP_SET(block_size, 16), 146 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), 147 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)), 148 QAT_SYM_AEAD_CAP(CHACHA20_POLY1305, 149 CAP_SET(block_size, 64), 150 CAP_RNG(key_size, 32, 32, 0), 151 CAP_RNG(digest_size, 16, 16, 0), 152 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 12, 12, 0)), 153 QAT_SYM_CIPHER_CAP(SM4_CBC, 154 CAP_SET(block_size, 16), 155 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), 156 QAT_SYM_CIPHER_CAP(SM4_CTR, 157 CAP_SET(block_size, 16), 158 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), 159 QAT_SYM_PLAIN_AUTH_CAP(SM3, 160 CAP_SET(block_size, 64), 161 CAP_RNG(digest_size, 32, 32, 0)), 162 QAT_SYM_AUTH_CAP(SM3_HMAC, 163 CAP_SET(block_size, 64), 164 CAP_RNG(key_size, 16, 64, 4), CAP_RNG(digest_size, 32, 32, 0), 165 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 166 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() 167 }; 168 169 static int 170 check_cipher_capa(const struct rte_cryptodev_capabilities *cap, 171 enum rte_crypto_cipher_algorithm algo) 172 { 173 if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 174 return 0; 175 if (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 176 return 0; 177 if (cap->sym.cipher.algo != algo) 178 return 0; 179 return 1; 180 } 181 182 static int 183 check_auth_capa(const struct rte_cryptodev_capabilities *cap, 184 enum rte_crypto_auth_algorithm algo) 185 { 186 if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 187 return 0; 188 if (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 189 return 0; 190 if (cap->sym.auth.algo != algo) 191 return 0; 192 return 1; 193 } 194 195 static int 196 qat_sym_crypto_cap_get_gen3(struct qat_cryptodev_private *internals, 197 const char *capa_memz_name, const uint16_t slice_map) 198 { 199 200 uint32_t i, iter = 0; 201 uint32_t curr_capa = 0; 202 uint32_t capa_num, legacy_capa_num; 203 uint32_t size = sizeof(qat_sym_crypto_caps_gen3); 204 uint32_t legacy_size = sizeof(qat_sym_crypto_legacy_caps_gen3); 205 capa_num = size/sizeof(struct rte_cryptodev_capabilities); 206 legacy_capa_num = legacy_size/sizeof(struct rte_cryptodev_capabilities); 207 struct rte_cryptodev_capabilities *cap; 208 209 if (unlikely(internals->qat_dev->options.legacy_alg)) 210 size = size + legacy_size; 211 212 internals->capa_mz = rte_memzone_lookup(capa_memz_name); 213 if (internals->capa_mz == NULL) { 214 internals->capa_mz = rte_memzone_reserve(capa_memz_name, 215 size, rte_socket_id(), 0); 216 if (internals->capa_mz == NULL) { 217 QAT_LOG(DEBUG, 218 "Error allocating memzone for capabilities"); 219 return -1; 220 } 221 } 222 223 struct rte_cryptodev_capabilities *addr = 224 (struct rte_cryptodev_capabilities *) 225 internals->capa_mz->addr; 226 struct rte_cryptodev_capabilities *capabilities; 227 228 if (unlikely(internals->qat_dev->options.legacy_alg)) { 229 capabilities = qat_sym_crypto_legacy_caps_gen3; 230 capa_num += legacy_capa_num; 231 } else { 232 capabilities = qat_sym_crypto_caps_gen3; 233 } 234 235 for (i = 0; i < capa_num; i++, iter++) { 236 if (unlikely(internals->qat_dev->options.legacy_alg) && 237 (i == legacy_capa_num)) { 238 capabilities = qat_sym_crypto_caps_gen3; 239 addr += curr_capa; 240 curr_capa = 0; 241 iter = 0; 242 } 243 244 if (slice_map & ICP_ACCEL_MASK_SM4_SLICE && ( 245 check_cipher_capa(&capabilities[iter], 246 RTE_CRYPTO_CIPHER_SM4_ECB) || 247 check_cipher_capa(&capabilities[iter], 248 RTE_CRYPTO_CIPHER_SM4_CBC) || 249 check_cipher_capa(&capabilities[iter], 250 RTE_CRYPTO_CIPHER_SM4_CTR))) { 251 continue; 252 } 253 if (slice_map & ICP_ACCEL_MASK_SM3_SLICE && ( 254 check_auth_capa(&capabilities[iter], 255 RTE_CRYPTO_AUTH_SM3) || 256 check_auth_capa(&capabilities[iter], 257 RTE_CRYPTO_AUTH_SM3_HMAC))) { 258 continue; 259 } 260 261 if (slice_map & ICP_ACCEL_MASK_ZUC_256_SLICE && ( 262 check_auth_capa(&capabilities[iter], 263 RTE_CRYPTO_AUTH_ZUC_EIA3) || 264 check_cipher_capa(&capabilities[iter], 265 RTE_CRYPTO_CIPHER_ZUC_EEA3))) { 266 continue; 267 } 268 269 if (internals->qat_dev->options.has_wireless_slice && ( 270 check_auth_capa(&capabilities[iter], 271 RTE_CRYPTO_AUTH_KASUMI_F9) || 272 check_cipher_capa(&capabilities[iter], 273 RTE_CRYPTO_CIPHER_KASUMI_F8) || 274 check_cipher_capa(&capabilities[iter], 275 RTE_CRYPTO_CIPHER_DES_CBC) || 276 check_cipher_capa(&capabilities[iter], 277 RTE_CRYPTO_CIPHER_DES_DOCSISBPI))) 278 continue; 279 280 memcpy(addr + curr_capa, capabilities + iter, 281 sizeof(struct rte_cryptodev_capabilities)); 282 283 if (internals->qat_dev->options.has_wireless_slice && ( 284 check_auth_capa(&capabilities[iter], 285 RTE_CRYPTO_AUTH_ZUC_EIA3))) { 286 cap = addr + curr_capa; 287 cap->sym.auth.key_size.max = 32; 288 cap->sym.auth.key_size.increment = 16; 289 cap->sym.auth.iv_size.max = 25; 290 cap->sym.auth.iv_size.increment = 1; 291 cap->sym.auth.digest_size.max = 16; 292 cap->sym.auth.digest_size.increment = 4; 293 } 294 if (internals->qat_dev->options.has_wireless_slice && ( 295 check_cipher_capa(&capabilities[iter], 296 RTE_CRYPTO_CIPHER_ZUC_EEA3))) { 297 cap = addr + curr_capa; 298 cap->sym.cipher.key_size.max = 32; 299 cap->sym.cipher.key_size.increment = 16; 300 cap->sym.cipher.iv_size.max = 25; 301 cap->sym.cipher.iv_size.increment = 1; 302 } 303 curr_capa++; 304 } 305 internals->qat_dev_capabilities = internals->capa_mz->addr; 306 307 return 0; 308 } 309 310 static __rte_always_inline void 311 enqueue_one_aead_job_gen3(struct qat_sym_session *ctx, 312 struct icp_qat_fw_la_bulk_req *req, 313 struct rte_crypto_va_iova_ptr *iv, 314 struct rte_crypto_va_iova_ptr *digest, 315 struct rte_crypto_va_iova_ptr *aad, 316 union rte_crypto_sym_ofs ofs, uint32_t data_len) 317 { 318 if (ctx->is_single_pass) { 319 struct icp_qat_fw_la_cipher_req_params *cipher_param = 320 (void *)&req->serv_specif_rqpars; 321 322 /* QAT GEN3 uses single pass to treat AEAD as 323 * cipher operation 324 */ 325 cipher_param = (void *)&req->serv_specif_rqpars; 326 327 qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req); 328 cipher_param->cipher_offset = ofs.ofs.cipher.head; 329 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head - 330 ofs.ofs.cipher.tail; 331 332 cipher_param->spc_aad_addr = aad->iova; 333 cipher_param->spc_auth_res_addr = digest->iova; 334 335 return; 336 } 337 338 enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len); 339 } 340 341 static __rte_always_inline void 342 enqueue_one_auth_job_gen3(struct qat_sym_session *ctx, 343 struct qat_sym_op_cookie *cookie, 344 struct icp_qat_fw_la_bulk_req *req, 345 struct rte_crypto_va_iova_ptr *digest, 346 struct rte_crypto_va_iova_ptr *auth_iv, 347 union rte_crypto_sym_ofs ofs, uint32_t data_len) 348 { 349 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl; 350 struct icp_qat_fw_la_cipher_req_params *cipher_param; 351 uint32_t ver_key_offset; 352 uint32_t auth_data_len = data_len - ofs.ofs.auth.head - 353 ofs.ofs.auth.tail; 354 355 if (!ctx->is_single_pass_gmac || 356 (auth_data_len > QAT_AES_GMAC_SPC_MAX_SIZE)) { 357 enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs, 358 data_len); 359 return; 360 } 361 362 cipher_cd_ctrl = (void *) &req->cd_ctrl; 363 cipher_param = (void *)&req->serv_specif_rqpars; 364 ver_key_offset = sizeof(struct icp_qat_hw_auth_setup) + 365 ICP_QAT_HW_GALOIS_128_STATE1_SZ + 366 ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ + 367 ICP_QAT_HW_GALOIS_E_CTR0_SZ + 368 sizeof(struct icp_qat_hw_cipher_config); 369 370 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || 371 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { 372 /* AES-GMAC */ 373 qat_set_cipher_iv(cipher_param, auth_iv, ctx->auth_iv.length, 374 req); 375 } 376 377 /* Fill separate Content Descriptor for this op */ 378 rte_memcpy(cookie->opt.spc_gmac.cd_cipher.key, 379 ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ? 380 ctx->cd.cipher.key : 381 RTE_PTR_ADD(&ctx->cd, ver_key_offset), 382 ctx->auth_key_length); 383 cookie->opt.spc_gmac.cd_cipher.cipher_config.val = 384 ICP_QAT_HW_CIPHER_CONFIG_BUILD( 385 ICP_QAT_HW_CIPHER_AEAD_MODE, 386 ctx->qat_cipher_alg, 387 ICP_QAT_HW_CIPHER_NO_CONVERT, 388 (ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ? 389 ICP_QAT_HW_CIPHER_ENCRYPT : 390 ICP_QAT_HW_CIPHER_DECRYPT)); 391 QAT_FIELD_SET(cookie->opt.spc_gmac.cd_cipher.cipher_config.val, 392 ctx->digest_length, 393 QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS, 394 QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK); 395 cookie->opt.spc_gmac.cd_cipher.cipher_config.reserved = 396 ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(auth_data_len); 397 398 /* Update the request */ 399 req->cd_pars.u.s.content_desc_addr = 400 cookie->opt.spc_gmac.cd_phys_addr; 401 req->cd_pars.u.s.content_desc_params_sz = RTE_ALIGN_CEIL( 402 sizeof(struct icp_qat_hw_cipher_config) + 403 ctx->auth_key_length, 8) >> 3; 404 req->comn_mid.src_length = data_len; 405 req->comn_mid.dst_length = 0; 406 407 cipher_param->spc_aad_addr = 0; 408 cipher_param->spc_auth_res_addr = digest->iova; 409 cipher_param->spc_aad_sz = auth_data_len; 410 cipher_param->reserved = 0; 411 cipher_param->spc_auth_res_sz = ctx->digest_length; 412 413 req->comn_hdr.service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; 414 cipher_cd_ctrl->cipher_cfg_offset = 0; 415 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); 416 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); 417 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 418 req->comn_hdr.serv_specif_flags, 419 ICP_QAT_FW_LA_SINGLE_PASS_PROTO); 420 ICP_QAT_FW_LA_PROTO_SET( 421 req->comn_hdr.serv_specif_flags, 422 ICP_QAT_FW_LA_NO_PROTO); 423 } 424 425 static int 426 qat_sym_build_op_aead_gen3(void *in_op, struct qat_sym_session *ctx, 427 uint8_t *out_msg, void *op_cookie) 428 { 429 register struct icp_qat_fw_la_bulk_req *req; 430 struct rte_crypto_op *op = in_op; 431 struct qat_sym_op_cookie *cookie = op_cookie; 432 struct rte_crypto_sgl in_sgl, out_sgl; 433 struct rte_crypto_vec in_vec[QAT_SYM_SGL_MAX_NUMBER], 434 out_vec[QAT_SYM_SGL_MAX_NUMBER]; 435 struct rte_crypto_va_iova_ptr cipher_iv; 436 struct rte_crypto_va_iova_ptr aad; 437 struct rte_crypto_va_iova_ptr digest; 438 union rte_crypto_sym_ofs ofs; 439 int32_t total_len; 440 441 in_sgl.vec = in_vec; 442 out_sgl.vec = out_vec; 443 444 req = (struct icp_qat_fw_la_bulk_req *)out_msg; 445 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); 446 447 ofs.raw = qat_sym_convert_op_to_vec_aead(op, ctx, &in_sgl, &out_sgl, 448 &cipher_iv, &aad, &digest); 449 if (unlikely(ofs.raw == UINT64_MAX)) { 450 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 451 return -EINVAL; 452 } 453 454 total_len = qat_sym_build_req_set_data(req, in_op, cookie, 455 in_sgl.vec, in_sgl.num, out_sgl.vec, out_sgl.num); 456 if (unlikely(total_len < 0)) { 457 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 458 return -EINVAL; 459 } 460 461 enqueue_one_aead_job_gen3(ctx, req, &cipher_iv, &digest, &aad, ofs, 462 total_len); 463 464 qat_sym_debug_log_dump(req, ctx, in_sgl.vec, in_sgl.num, &cipher_iv, 465 NULL, &aad, &digest); 466 467 return 0; 468 } 469 470 static int 471 qat_sym_build_op_auth_gen3(void *in_op, struct qat_sym_session *ctx, 472 uint8_t *out_msg, void *op_cookie) 473 { 474 register struct icp_qat_fw_la_bulk_req *req; 475 struct rte_crypto_op *op = in_op; 476 struct qat_sym_op_cookie *cookie = op_cookie; 477 struct rte_crypto_sgl in_sgl, out_sgl; 478 struct rte_crypto_vec in_vec[QAT_SYM_SGL_MAX_NUMBER], 479 out_vec[QAT_SYM_SGL_MAX_NUMBER]; 480 struct rte_crypto_va_iova_ptr auth_iv; 481 struct rte_crypto_va_iova_ptr digest; 482 union rte_crypto_sym_ofs ofs; 483 int32_t total_len; 484 485 in_sgl.vec = in_vec; 486 out_sgl.vec = out_vec; 487 488 req = (struct icp_qat_fw_la_bulk_req *)out_msg; 489 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); 490 491 ofs.raw = qat_sym_convert_op_to_vec_auth(op, ctx, &in_sgl, &out_sgl, 492 NULL, &auth_iv, &digest, op_cookie); 493 if (unlikely(ofs.raw == UINT64_MAX)) { 494 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 495 return -EINVAL; 496 } 497 498 total_len = qat_sym_build_req_set_data(req, in_op, cookie, 499 in_sgl.vec, in_sgl.num, out_sgl.vec, out_sgl.num); 500 if (unlikely(total_len < 0)) { 501 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 502 return -EINVAL; 503 } 504 505 enqueue_one_auth_job_gen3(ctx, cookie, req, &digest, &auth_iv, 506 ofs, total_len); 507 508 qat_sym_debug_log_dump(req, ctx, in_sgl.vec, in_sgl.num, NULL, 509 &auth_iv, NULL, &digest); 510 511 return 0; 512 } 513 514 static int 515 qat_sym_crypto_set_session_gen3(void *cdev, void *session) 516 { 517 struct qat_sym_session *ctx = session; 518 enum rte_proc_type_t proc_type = rte_eal_process_type(); 519 int ret; 520 struct qat_cryptodev_private *internals; 521 522 internals = ((struct rte_cryptodev *)cdev)->data->dev_private; 523 524 if (proc_type == RTE_PROC_AUTO || proc_type == RTE_PROC_INVALID) 525 return -EINVAL; 526 527 ret = qat_sym_crypto_set_session_gen1(cdev, session); 528 /* special single pass build request for GEN3 */ 529 if (ctx->is_single_pass) 530 ctx->build_request[proc_type] = qat_sym_build_op_aead_gen3; 531 else if (ctx->is_single_pass_gmac) 532 ctx->build_request[proc_type] = qat_sym_build_op_auth_gen3; 533 534 if (ret == -ENOTSUP) { 535 /* GEN1 returning -ENOTSUP as it cannot handle some mixed algo, 536 * this is addressed by GEN3 537 */ 538 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 && 539 ctx->qat_cipher_alg != 540 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { 541 qat_sym_session_set_ext_hash_flags_gen2(ctx, 542 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS); 543 } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 && 544 ctx->qat_cipher_alg != 545 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { 546 qat_sym_session_set_ext_hash_flags_gen2(ctx, 547 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS); 548 } else if ((ctx->aes_cmac || 549 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) && 550 (ctx->qat_cipher_alg == 551 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || 552 ctx->qat_cipher_alg == 553 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) { 554 qat_sym_session_set_ext_hash_flags_gen2(ctx, 0); 555 } else if ((internals->qat_dev->options.has_wireless_slice) && 556 ((ctx->aes_cmac || 557 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) && 558 (ctx->qat_cipher_alg == 559 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || 560 ctx->qat_cipher_alg == 561 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3 || 562 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_256))) { 563 qat_sym_session_set_ext_hash_flags_gen2(ctx, 0); 564 } else if ((internals->qat_dev->options.has_wireless_slice) && 565 (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_256_MAC_32 || 566 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_256_MAC_64 || 567 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_256_MAC_128) && 568 ctx->qat_cipher_alg != ICP_QAT_HW_CIPHER_ALGO_ZUC_256) { 569 qat_sym_session_set_ext_hash_flags_gen2(ctx, 570 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS); 571 } 572 573 ret = 0; 574 } 575 576 return ret; 577 } 578 579 static int 580 qat_sym_dp_enqueue_single_aead_gen3(void *qp_data, uint8_t *drv_ctx, 581 struct rte_crypto_vec *data, uint16_t n_data_vecs, 582 union rte_crypto_sym_ofs ofs, 583 struct rte_crypto_va_iova_ptr *iv, 584 struct rte_crypto_va_iova_ptr *digest, 585 struct rte_crypto_va_iova_ptr *aad, 586 void *user_data) 587 { 588 struct qat_qp *qp = qp_data; 589 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; 590 struct qat_queue *tx_queue = &qp->tx_q; 591 struct qat_sym_op_cookie *cookie; 592 struct qat_sym_session *ctx = dp_ctx->session; 593 struct icp_qat_fw_la_bulk_req *req; 594 595 int32_t data_len; 596 uint32_t tail = dp_ctx->tail; 597 598 req = (struct icp_qat_fw_la_bulk_req *)( 599 (uint8_t *)tx_queue->base_addr + tail); 600 cookie = qp->op_cookies[tail >> tx_queue->trailz]; 601 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; 602 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); 603 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail); 604 data_len = qat_sym_build_req_set_data(req, user_data, cookie, 605 data, n_data_vecs, NULL, 0); 606 if (unlikely(data_len < 0)) 607 return -1; 608 609 enqueue_one_aead_job_gen3(ctx, req, iv, digest, aad, ofs, 610 (uint32_t)data_len); 611 612 dp_ctx->tail = tail; 613 dp_ctx->cached_enqueue++; 614 615 qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv, 616 NULL, aad, digest); 617 618 return 0; 619 } 620 621 static uint32_t 622 qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx, 623 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, 624 void *user_data[], int *status) 625 { 626 struct qat_qp *qp = qp_data; 627 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; 628 struct qat_queue *tx_queue = &qp->tx_q; 629 struct qat_sym_session *ctx = dp_ctx->session; 630 uint32_t i, n; 631 uint32_t tail; 632 struct icp_qat_fw_la_bulk_req *req; 633 int32_t data_len; 634 635 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num); 636 if (unlikely(n == 0)) { 637 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num); 638 *status = 0; 639 return 0; 640 } 641 642 tail = dp_ctx->tail; 643 644 for (i = 0; i < n; i++) { 645 struct qat_sym_op_cookie *cookie = 646 qp->op_cookies[tail >> tx_queue->trailz]; 647 648 req = (struct icp_qat_fw_la_bulk_req *)( 649 (uint8_t *)tx_queue->base_addr + tail); 650 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); 651 652 if (vec->dest_sgl) { 653 data_len = qat_sym_build_req_set_data(req, 654 user_data[i], cookie, 655 vec->src_sgl[i].vec, vec->src_sgl[i].num, 656 vec->dest_sgl[i].vec, vec->dest_sgl[i].num); 657 } else { 658 data_len = qat_sym_build_req_set_data(req, 659 user_data[i], cookie, 660 vec->src_sgl[i].vec, 661 vec->src_sgl[i].num, NULL, 0); 662 } 663 664 if (unlikely(data_len < 0)) 665 break; 666 667 enqueue_one_aead_job_gen3(ctx, req, &vec->iv[i], 668 &vec->digest[i], &vec->aad[i], ofs, 669 (uint32_t)data_len); 670 671 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; 672 673 qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec, 674 vec->src_sgl[i].num, &vec->iv[i], NULL, 675 &vec->aad[i], &vec->digest[i]); 676 } 677 678 if (unlikely(i < n)) 679 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i); 680 681 dp_ctx->tail = tail; 682 dp_ctx->cached_enqueue += i; 683 *status = 0; 684 return i; 685 } 686 687 static int 688 qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx, 689 struct rte_crypto_vec *data, uint16_t n_data_vecs, 690 union rte_crypto_sym_ofs ofs, 691 struct rte_crypto_va_iova_ptr *iv __rte_unused, 692 struct rte_crypto_va_iova_ptr *digest, 693 struct rte_crypto_va_iova_ptr *auth_iv, 694 void *user_data) 695 { 696 struct qat_qp *qp = qp_data; 697 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; 698 struct qat_queue *tx_queue = &qp->tx_q; 699 struct qat_sym_op_cookie *cookie; 700 struct qat_sym_session *ctx = dp_ctx->session; 701 struct icp_qat_fw_la_bulk_req *req; 702 int32_t data_len; 703 uint32_t tail = dp_ctx->tail; 704 struct rte_crypto_va_iova_ptr null_digest; 705 struct rte_crypto_va_iova_ptr *job_digest = digest; 706 707 req = (struct icp_qat_fw_la_bulk_req *)( 708 (uint8_t *)tx_queue->base_addr + tail); 709 cookie = qp->op_cookies[tail >> tx_queue->trailz]; 710 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; 711 712 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); 713 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail); 714 data_len = qat_sym_build_req_set_data(req, user_data, cookie, 715 data, n_data_vecs, NULL, 0); 716 if (unlikely(data_len < 0)) 717 return -1; 718 719 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { 720 null_digest.iova = cookie->digest_null_phys_addr; 721 job_digest = &null_digest; 722 } 723 724 enqueue_one_auth_job_gen3(ctx, cookie, req, job_digest, auth_iv, ofs, 725 (uint32_t)data_len); 726 727 dp_ctx->tail = tail; 728 dp_ctx->cached_enqueue++; 729 730 return 0; 731 } 732 733 static uint32_t 734 qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx, 735 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, 736 void *user_data[], int *status) 737 { 738 struct qat_qp *qp = qp_data; 739 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; 740 struct qat_queue *tx_queue = &qp->tx_q; 741 struct qat_sym_session *ctx = dp_ctx->session; 742 uint32_t i, n; 743 uint32_t tail; 744 struct icp_qat_fw_la_bulk_req *req; 745 int32_t data_len; 746 struct rte_crypto_va_iova_ptr null_digest; 747 struct rte_crypto_va_iova_ptr *job_digest = NULL; 748 749 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num); 750 if (unlikely(n == 0)) { 751 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num); 752 *status = 0; 753 return 0; 754 } 755 756 tail = dp_ctx->tail; 757 758 for (i = 0; i < n; i++) { 759 struct qat_sym_op_cookie *cookie = 760 qp->op_cookies[tail >> tx_queue->trailz]; 761 762 req = (struct icp_qat_fw_la_bulk_req *)( 763 (uint8_t *)tx_queue->base_addr + tail); 764 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); 765 766 if (vec->dest_sgl) { 767 data_len = qat_sym_build_req_set_data(req, 768 user_data[i], cookie, 769 vec->src_sgl[i].vec, vec->src_sgl[i].num, 770 vec->dest_sgl[i].vec, vec->dest_sgl[i].num); 771 } else { 772 data_len = qat_sym_build_req_set_data(req, 773 user_data[i], cookie, 774 vec->src_sgl[i].vec, 775 vec->src_sgl[i].num, NULL, 0); 776 } 777 778 if (unlikely(data_len < 0)) 779 break; 780 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { 781 null_digest.iova = cookie->digest_null_phys_addr; 782 job_digest = &null_digest; 783 } else 784 job_digest = &vec->digest[i]; 785 786 enqueue_one_auth_job_gen3(ctx, cookie, req, job_digest, 787 &vec->auth_iv[i], ofs, (uint32_t)data_len); 788 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; 789 } 790 791 if (unlikely(i < n)) 792 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i); 793 794 dp_ctx->tail = tail; 795 dp_ctx->cached_enqueue += i; 796 *status = 0; 797 return i; 798 } 799 800 static int 801 qat_sym_configure_raw_dp_ctx_gen3(void *_raw_dp_ctx, void *_ctx) 802 { 803 struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx; 804 struct qat_sym_session *ctx = _ctx; 805 int ret; 806 807 ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx); 808 if (ret < 0) 809 return ret; 810 811 if (ctx->is_single_pass) { 812 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen3; 813 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen3; 814 } else if (ctx->is_single_pass_gmac) { 815 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen3; 816 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen3; 817 } 818 819 return 0; 820 } 821 822 823 RTE_INIT(qat_sym_crypto_gen3_init) 824 { 825 qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1; 826 qat_sym_gen_dev_ops[QAT_GEN3].get_capabilities = 827 qat_sym_crypto_cap_get_gen3; 828 qat_sym_gen_dev_ops[QAT_GEN3].get_feature_flags = 829 qat_sym_crypto_feature_flags_get_gen1; 830 qat_sym_gen_dev_ops[QAT_GEN3].set_session = 831 qat_sym_crypto_set_session_gen3; 832 qat_sym_gen_dev_ops[QAT_GEN3].set_raw_dp_ctx = 833 qat_sym_configure_raw_dp_ctx_gen3; 834 qat_sym_gen_dev_ops[QAT_GEN3].create_security_ctx = 835 qat_sym_create_security_gen1; 836 } 837 838 RTE_INIT(qat_asym_crypto_gen3_init) 839 { 840 qat_asym_gen_dev_ops[QAT_GEN3].cryptodev_ops = 841 &qat_asym_crypto_ops_gen1; 842 qat_asym_gen_dev_ops[QAT_GEN3].get_capabilities = 843 qat_asym_crypto_cap_get_gen1; 844 qat_asym_gen_dev_ops[QAT_GEN3].get_feature_flags = 845 qat_asym_crypto_feature_flags_get_gen1; 846 qat_asym_gen_dev_ops[QAT_GEN3].set_session = 847 qat_asym_crypto_set_session_gen1; 848 } 849