1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2022 Intel Corporation 3 */ 4 5 #include <rte_cryptodev.h> 6 #include <cryptodev_pmd.h> 7 #include "qat_sym_session.h" 8 #include "qat_sym.h" 9 #include "qat_asym.h" 10 #include "qat_crypto.h" 11 #include "qat_crypto_pmd_gens.h" 12 13 #define MIXED_CRYPTO_MIN_FW_VER 0x04090000 14 15 static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen2[] = { 16 QAT_SYM_CIPHER_CAP(DES_CBC, 17 CAP_SET(block_size, 8), 18 CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), 19 QAT_SYM_CIPHER_CAP(3DES_CBC, 20 CAP_SET(block_size, 8), 21 CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), 22 QAT_SYM_CIPHER_CAP(3DES_CTR, 23 CAP_SET(block_size, 8), 24 CAP_RNG(key_size, 16, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), 25 QAT_SYM_PLAIN_AUTH_CAP(SHA1, 26 CAP_SET(block_size, 64), 27 CAP_RNG(digest_size, 1, 20, 1)), 28 QAT_SYM_AUTH_CAP(SHA224, 29 CAP_SET(block_size, 64), 30 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1), 31 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 32 QAT_SYM_AUTH_CAP(SHA224_HMAC, 33 CAP_SET(block_size, 64), 34 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1), 35 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 36 QAT_SYM_AUTH_CAP(SHA1_HMAC, 37 CAP_SET(block_size, 64), 38 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1), 39 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 40 QAT_SYM_AUTH_CAP(MD5_HMAC, 41 CAP_SET(block_size, 64), 42 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 16, 1), 43 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 44 QAT_SYM_CIPHER_CAP(DES_DOCSISBPI, 45 CAP_SET(block_size, 8), 46 CAP_RNG(key_size, 8, 8, 0), CAP_RNG(iv_size, 8, 8, 0)), 47 }; 48 49 static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen2[] = { 50 QAT_SYM_AEAD_CAP(AES_GCM, 51 CAP_SET(block_size, 16), 52 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), 53 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)), 54 QAT_SYM_AEAD_CAP(AES_CCM, 55 CAP_SET(block_size, 16), 56 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2), 57 CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)), 58 QAT_SYM_AUTH_CAP(AES_GMAC, 59 CAP_SET(block_size, 16), 60 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), 61 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)), 62 QAT_SYM_AUTH_CAP(AES_CMAC, 63 CAP_SET(block_size, 16), 64 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4), 65 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 66 QAT_SYM_AUTH_CAP(SHA256, 67 CAP_SET(block_size, 64), 68 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1), 69 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 70 QAT_SYM_AUTH_CAP(SHA384, 71 CAP_SET(block_size, 128), 72 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1), 73 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 74 QAT_SYM_AUTH_CAP(SHA512, 75 CAP_SET(block_size, 128), 76 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1), 77 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 78 QAT_SYM_PLAIN_AUTH_CAP(SHA3_256, 79 CAP_SET(block_size, 136), 80 CAP_RNG(digest_size, 32, 32, 0)), 81 QAT_SYM_AUTH_CAP(SHA256_HMAC, 82 CAP_SET(block_size, 64), 83 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1), 84 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 85 QAT_SYM_AUTH_CAP(SHA384_HMAC, 86 CAP_SET(block_size, 128), 87 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1), 88 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 89 QAT_SYM_AUTH_CAP(SHA512_HMAC, 90 CAP_SET(block_size, 128), 91 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1), 92 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 93 QAT_SYM_AUTH_CAP(AES_XCBC_MAC, 94 CAP_SET(block_size, 16), 95 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0), 96 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 97 QAT_SYM_AUTH_CAP(SNOW3G_UIA2, 98 CAP_SET(block_size, 16), 99 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), 100 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)), 101 QAT_SYM_AUTH_CAP(KASUMI_F9, 102 CAP_SET(block_size, 8), 103 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), 104 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 105 QAT_SYM_AUTH_CAP(NULL, 106 CAP_SET(block_size, 1), 107 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size), 108 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), 109 QAT_SYM_CIPHER_CAP(AES_CBC, 110 CAP_SET(block_size, 16), 111 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), 112 QAT_SYM_CIPHER_CAP(AES_CTR, 113 CAP_SET(block_size, 16), 114 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), 115 QAT_SYM_CIPHER_CAP(AES_XTS, 116 CAP_SET(block_size, 16), 117 CAP_RNG(key_size, 32, 64, 32), CAP_RNG(iv_size, 16, 16, 0)), 118 QAT_SYM_CIPHER_CAP(AES_DOCSISBPI, 119 CAP_SET(block_size, 16), 120 CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)), 121 QAT_SYM_CIPHER_CAP(SNOW3G_UEA2, 122 CAP_SET(block_size, 16), 123 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), 124 QAT_SYM_CIPHER_CAP(KASUMI_F8, 125 CAP_SET(block_size, 8), 126 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 8, 8, 0)), 127 QAT_SYM_CIPHER_CAP(NULL, 128 CAP_SET(block_size, 1), 129 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)), 130 QAT_SYM_CIPHER_CAP(ZUC_EEA3, 131 CAP_SET(block_size, 16), 132 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), 133 QAT_SYM_AUTH_CAP(ZUC_EIA3, 134 CAP_SET(block_size, 16), 135 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), 136 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)), 137 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() 138 }; 139 140 static int 141 qat_sym_crypto_qp_setup_gen2(struct rte_cryptodev *dev, uint16_t qp_id, 142 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id) 143 { 144 struct qat_cryptodev_private *qat_sym_private = dev->data->dev_private; 145 struct qat_qp *qp; 146 int ret; 147 148 if (qat_cryptodev_qp_setup(dev, qp_id, qp_conf, socket_id)) { 149 QAT_LOG(DEBUG, "QAT qp setup failed"); 150 return -1; 151 } 152 153 qp = qat_sym_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][qp_id]; 154 ret = qat_cq_get_fw_version(qp); 155 if (ret < 0) { 156 qat_cryptodev_qp_release(dev, qp_id); 157 return ret; 158 } 159 160 if (ret != 0) 161 QAT_LOG(DEBUG, "QAT firmware version: %d.%d.%d", 162 (ret >> 24) & 0xff, 163 (ret >> 16) & 0xff, 164 (ret >> 8) & 0xff); 165 else 166 QAT_LOG(DEBUG, "unknown QAT firmware version"); 167 168 /* set capabilities based on the fw version */ 169 qat_sym_private->internal_capabilities |= QAT_SYM_CAP_VALID | 170 ((ret >= MIXED_CRYPTO_MIN_FW_VER) ? 171 QAT_SYM_CAP_MIXED_CRYPTO : 0); 172 return 0; 173 } 174 175 void 176 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session, 177 uint8_t hash_flag) 178 { 179 struct icp_qat_fw_comn_req_hdr *header = &session->fw_req.comn_hdr; 180 struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *cd_ctrl = 181 (struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *) 182 session->fw_req.cd_ctrl.content_desc_ctrl_lw; 183 184 /* Set the Use Extended Protocol Flags bit in LW 1 */ 185 ICP_QAT_FW_USE_EXTENDED_PROTOCOL_FLAGS_SET( 186 header->ext_flags, QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS); 187 188 /* Set Hash Flags in LW 28 */ 189 cd_ctrl->hash_flags |= hash_flag; 190 191 /* Set proto flags in LW 1 */ 192 switch (session->qat_cipher_alg) { 193 case ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2: 194 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, 195 ICP_QAT_FW_LA_SNOW_3G_PROTO); 196 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( 197 header->serv_specif_flags, 0); 198 break; 199 case ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3: 200 case ICP_QAT_HW_CIPHER_ALGO_ZUC_256: 201 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, 202 ICP_QAT_FW_LA_NO_PROTO); 203 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( 204 header->serv_specif_flags, 205 ICP_QAT_FW_LA_ZUC_3G_PROTO); 206 break; 207 default: 208 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, 209 ICP_QAT_FW_LA_NO_PROTO); 210 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( 211 header->serv_specif_flags, 0); 212 break; 213 } 214 } 215 216 static int 217 qat_sym_crypto_set_session_gen2(void *cdev, void *session) 218 { 219 struct rte_cryptodev *dev = cdev; 220 struct qat_sym_session *ctx = session; 221 const struct qat_cryptodev_private *qat_private = 222 dev->data->dev_private; 223 int ret; 224 225 ret = qat_sym_crypto_set_session_gen1(cdev, session); 226 if (ret == -ENOTSUP) { 227 /* GEN1 returning -ENOTSUP as it cannot handle some mixed algo, 228 * but some are not supported by GEN2, so checking here 229 */ 230 if ((qat_private->internal_capabilities & 231 QAT_SYM_CAP_MIXED_CRYPTO) == 0) 232 return -ENOTSUP; 233 234 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 && 235 ctx->qat_cipher_alg != 236 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { 237 qat_sym_session_set_ext_hash_flags_gen2(ctx, 238 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS); 239 } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 && 240 ctx->qat_cipher_alg != 241 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { 242 qat_sym_session_set_ext_hash_flags_gen2(ctx, 243 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS); 244 } else if ((ctx->aes_cmac || 245 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) && 246 (ctx->qat_cipher_alg == 247 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || 248 ctx->qat_cipher_alg == 249 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) { 250 qat_sym_session_set_ext_hash_flags_gen2(ctx, 0); 251 } 252 253 ret = 0; 254 } 255 256 return ret; 257 } 258 259 struct rte_cryptodev_ops qat_sym_crypto_ops_gen2 = { 260 261 /* Device related operations */ 262 .dev_configure = qat_cryptodev_config, 263 .dev_start = qat_cryptodev_start, 264 .dev_stop = qat_cryptodev_stop, 265 .dev_close = qat_cryptodev_close, 266 .dev_infos_get = qat_cryptodev_info_get, 267 268 .stats_get = qat_cryptodev_stats_get, 269 .stats_reset = qat_cryptodev_stats_reset, 270 .queue_pair_setup = qat_sym_crypto_qp_setup_gen2, 271 .queue_pair_release = qat_cryptodev_qp_release, 272 273 /* Crypto related operations */ 274 .sym_session_get_size = qat_sym_session_get_private_size, 275 .sym_session_configure = qat_sym_session_configure, 276 .sym_session_clear = qat_sym_session_clear, 277 278 /* Raw data-path API related operations */ 279 .sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size, 280 .sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx, 281 }; 282 283 static int 284 qat_sym_crypto_cap_get_gen2(struct qat_cryptodev_private *internals, 285 const char *capa_memz_name, 286 const uint16_t __rte_unused slice_map) 287 { 288 uint32_t legacy_capa_num; 289 uint32_t size = sizeof(qat_sym_crypto_caps_gen2); 290 uint32_t legacy_size = sizeof(qat_sym_crypto_legacy_caps_gen2); 291 legacy_capa_num = legacy_size/sizeof(struct rte_cryptodev_capabilities); 292 293 if (unlikely(qat_legacy_capa)) 294 size = size + legacy_size; 295 296 internals->capa_mz = rte_memzone_lookup(capa_memz_name); 297 if (internals->capa_mz == NULL) { 298 internals->capa_mz = rte_memzone_reserve(capa_memz_name, 299 size, rte_socket_id(), 0); 300 if (internals->capa_mz == NULL) { 301 QAT_LOG(DEBUG, 302 "Error allocating memzone for capabilities"); 303 return -1; 304 } 305 } 306 307 struct rte_cryptodev_capabilities *addr = 308 (struct rte_cryptodev_capabilities *) 309 internals->capa_mz->addr; 310 struct rte_cryptodev_capabilities *capabilities; 311 312 if (unlikely(qat_legacy_capa)) { 313 capabilities = qat_sym_crypto_legacy_caps_gen2; 314 memcpy(addr, capabilities, legacy_size); 315 addr += legacy_capa_num; 316 } 317 capabilities = qat_sym_crypto_caps_gen2; 318 memcpy(addr, capabilities, sizeof(qat_sym_crypto_caps_gen2)); 319 internals->qat_dev_capabilities = internals->capa_mz->addr; 320 321 return 0; 322 } 323 324 RTE_INIT(qat_sym_crypto_gen2_init) 325 { 326 qat_sym_gen_dev_ops[QAT_GEN2].cryptodev_ops = &qat_sym_crypto_ops_gen2; 327 qat_sym_gen_dev_ops[QAT_GEN2].get_capabilities = 328 qat_sym_crypto_cap_get_gen2; 329 qat_sym_gen_dev_ops[QAT_GEN2].set_session = 330 qat_sym_crypto_set_session_gen2; 331 qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx = 332 qat_sym_configure_raw_dp_ctx_gen1; 333 qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags = 334 qat_sym_crypto_feature_flags_get_gen1; 335 qat_sym_gen_dev_ops[QAT_GEN2].create_security_ctx = 336 qat_sym_create_security_gen1; 337 } 338 339 RTE_INIT(qat_asym_crypto_gen2_init) 340 { 341 qat_asym_gen_dev_ops[QAT_GEN2].cryptodev_ops = 342 &qat_asym_crypto_ops_gen1; 343 qat_asym_gen_dev_ops[QAT_GEN2].get_capabilities = 344 qat_asym_crypto_cap_get_gen1; 345 qat_asym_gen_dev_ops[QAT_GEN2].get_feature_flags = 346 qat_asym_crypto_feature_flags_get_gen1; 347 qat_asym_gen_dev_ops[QAT_GEN2].set_session = 348 qat_asym_crypto_set_session_gen1; 349 } 350