1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2023 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_eal_paging.h> 8 #include <rte_errno.h> 9 #include <rte_log.h> 10 #include <bus_pci_driver.h> 11 #include <rte_memory.h> 12 #include <rte_io.h> 13 14 #include <mlx5_glue.h> 15 #include <mlx5_common.h> 16 #include <mlx5_devx_cmds.h> 17 #include <mlx5_common_os.h> 18 19 #include "mlx5_crypto_utils.h" 20 #include "mlx5_crypto.h" 21 22 /* 23 * AES-GCM uses indirect KLM mode. The UMR WQE comprises of WQE control + 24 * UMR control + mkey context + indirect KLM. The WQE size is aligned to 25 * be 3 WQEBBS. 26 */ 27 #define MLX5_UMR_GCM_WQE_SIZE \ 28 (RTE_ALIGN(sizeof(struct mlx5_umr_wqe) + sizeof(struct mlx5_wqe_dseg), \ 29 MLX5_SEND_WQE_BB)) 30 31 #define MLX5_UMR_GCM_WQE_SET_SIZE \ 32 (MLX5_UMR_GCM_WQE_SIZE + \ 33 RTE_ALIGN(sizeof(struct mlx5_wqe_send_en_wqe), \ 34 MLX5_SEND_WQE_BB)) 35 36 #define MLX5_UMR_GCM_WQE_STRIDE \ 37 (MLX5_UMR_GCM_WQE_SIZE / MLX5_SEND_WQE_BB) 38 39 #define MLX5_MMO_CRYPTO_OPC (MLX5_OPCODE_MMO | \ 40 (MLX5_OPC_MOD_MMO_CRYPTO << WQE_CSEG_OPC_MOD_OFFSET)) 41 42 /* 43 * The status default value is RTE_CRYPTO_OP_STATUS_SUCCESS. 44 * Copy tag should fill different value to status. 45 */ 46 #define MLX5_CRYPTO_OP_STATUS_GCM_TAG_COPY (RTE_CRYPTO_OP_STATUS_SUCCESS + 1) 47 48 struct mlx5_crypto_gcm_op_info { 49 bool need_umr; 50 bool is_oop; 51 bool is_enc; 52 void *digest; 53 void *src_addr; 54 }; 55 56 struct mlx5_crypto_gcm_data { 57 void *src_addr; 58 uint32_t src_bytes; 59 void *dst_addr; 60 uint32_t dst_bytes; 61 uint32_t src_mkey; 62 uint32_t dst_mkey; 63 }; 64 65 struct mlx5_crypto_gcm_tag_cpy_info { 66 void *digest; 67 uint8_t tag_len; 68 } __rte_packed; 69 70 static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = { 71 { 72 .op = RTE_CRYPTO_OP_TYPE_UNDEFINED, 73 }, 74 { 75 .op = RTE_CRYPTO_OP_TYPE_UNDEFINED, 76 } 77 }; 78 79 int 80 mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek, 81 struct mlx5_devx_dek_attr *dek_attr, 82 void *cb_ctx) 83 { 84 uint32_t offset = 0; 85 struct mlx5_crypto_dek_ctx *ctx = cb_ctx; 86 struct rte_crypto_aead_xform *aead_ctx = &ctx->xform->aead; 87 88 if (aead_ctx->algo != RTE_CRYPTO_AEAD_AES_GCM) { 89 DRV_LOG(ERR, "Only AES-GCM algo supported."); 90 return -EINVAL; 91 } 92 dek_attr->key_purpose = MLX5_CRYPTO_KEY_PURPOSE_GCM; 93 switch (aead_ctx->key.length) { 94 case 16: 95 offset = 16; 96 dek->size = 16; 97 dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_128b; 98 break; 99 case 32: 100 dek->size = 32; 101 dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_256b; 102 break; 103 default: 104 DRV_LOG(ERR, "Wrapped key size not supported."); 105 return -EINVAL; 106 } 107 memcpy(&dek_attr->key[offset], aead_ctx->key.data, aead_ctx->key.length); 108 memcpy(&dek->data, aead_ctx->key.data, aead_ctx->key.length); 109 return 0; 110 } 111 112 static int 113 mlx5_crypto_generate_gcm_cap(struct mlx5_hca_crypto_mmo_attr *mmo_attr, 114 struct rte_cryptodev_capabilities *cap) 115 { 116 /* Init key size. */ 117 if (mmo_attr->gcm_128_encrypt && mmo_attr->gcm_128_decrypt && 118 mmo_attr->gcm_256_encrypt && mmo_attr->gcm_256_decrypt) { 119 cap->sym.aead.key_size.min = 16; 120 cap->sym.aead.key_size.max = 32; 121 cap->sym.aead.key_size.increment = 16; 122 } else if (mmo_attr->gcm_256_encrypt && mmo_attr->gcm_256_decrypt) { 123 cap->sym.aead.key_size.min = 32; 124 cap->sym.aead.key_size.max = 32; 125 cap->sym.aead.key_size.increment = 0; 126 } else if (mmo_attr->gcm_128_encrypt && mmo_attr->gcm_128_decrypt) { 127 cap->sym.aead.key_size.min = 16; 128 cap->sym.aead.key_size.max = 16; 129 cap->sym.aead.key_size.increment = 0; 130 } else { 131 DRV_LOG(ERR, "No available AES-GCM encryption/decryption supported."); 132 return -1; 133 } 134 /* Init tag size. */ 135 if (mmo_attr->gcm_auth_tag_128 && mmo_attr->gcm_auth_tag_96) { 136 cap->sym.aead.digest_size.min = 12; 137 cap->sym.aead.digest_size.max = 16; 138 cap->sym.aead.digest_size.increment = 4; 139 } else if (mmo_attr->gcm_auth_tag_96) { 140 cap->sym.aead.digest_size.min = 12; 141 cap->sym.aead.digest_size.max = 12; 142 cap->sym.aead.digest_size.increment = 0; 143 } else if (mmo_attr->gcm_auth_tag_128) { 144 cap->sym.aead.digest_size.min = 16; 145 cap->sym.aead.digest_size.max = 16; 146 cap->sym.aead.digest_size.increment = 0; 147 } else { 148 DRV_LOG(ERR, "No available AES-GCM tag size supported."); 149 return -1; 150 } 151 /* Init AAD size. */ 152 cap->sym.aead.aad_size.min = 0; 153 cap->sym.aead.aad_size.max = UINT16_MAX; 154 cap->sym.aead.aad_size.increment = 1; 155 /* Init IV size. */ 156 cap->sym.aead.iv_size.min = 12; 157 cap->sym.aead.iv_size.max = 12; 158 cap->sym.aead.iv_size.increment = 0; 159 /* Init left items. */ 160 cap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 161 cap->sym.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD; 162 cap->sym.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; 163 return 0; 164 } 165 166 static int 167 mlx5_crypto_sym_gcm_session_configure(struct rte_cryptodev *dev, 168 struct rte_crypto_sym_xform *xform, 169 struct rte_cryptodev_sym_session *session) 170 { 171 struct mlx5_crypto_priv *priv = dev->data->dev_private; 172 struct mlx5_crypto_session *sess_private_data = CRYPTODEV_GET_SYM_SESS_PRIV(session); 173 struct rte_crypto_aead_xform *aead = &xform->aead; 174 uint32_t op_type; 175 176 if (unlikely(xform->next != NULL)) { 177 DRV_LOG(ERR, "Xform next is not supported."); 178 return -ENOTSUP; 179 } 180 if (aead->algo != RTE_CRYPTO_AEAD_AES_GCM) { 181 DRV_LOG(ERR, "Only AES-GCM algorithm is supported."); 182 return -ENOTSUP; 183 } 184 if (aead->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 185 op_type = MLX5_CRYPTO_OP_TYPE_ENCRYPTION; 186 else 187 op_type = MLX5_CRYPTO_OP_TYPE_DECRYPTION; 188 sess_private_data->op_type = op_type; 189 sess_private_data->mmo_ctrl = rte_cpu_to_be_32 190 (op_type << MLX5_CRYPTO_MMO_OP_OFFSET | 191 MLX5_ENCRYPTION_TYPE_AES_GCM << MLX5_CRYPTO_MMO_TYPE_OFFSET); 192 sess_private_data->wqe_aad_len = rte_cpu_to_be_32((uint32_t)aead->aad_length); 193 sess_private_data->wqe_tag_len = rte_cpu_to_be_32((uint32_t)aead->digest_length); 194 sess_private_data->aad_len = aead->aad_length; 195 sess_private_data->tag_len = aead->digest_length; 196 sess_private_data->iv_offset = aead->iv.offset; 197 sess_private_data->iv_len = aead->iv.length; 198 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, xform); 199 if (sess_private_data->dek == NULL) { 200 DRV_LOG(ERR, "Failed to prepare dek."); 201 return -ENOMEM; 202 } 203 sess_private_data->dek_id = 204 rte_cpu_to_be_32(sess_private_data->dek->obj->id & 205 0xffffff); 206 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data); 207 return 0; 208 } 209 210 static void * 211 mlx5_crypto_gcm_mkey_klm_update(struct mlx5_crypto_priv *priv, 212 struct mlx5_crypto_qp *qp __rte_unused, 213 uint32_t idx) 214 { 215 return &qp->klm_array[idx * priv->max_klm_num]; 216 } 217 218 static int 219 mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) 220 { 221 struct mlx5_crypto_priv *priv = dev->data->dev_private; 222 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 223 224 if (qp->umr_qp_obj.qp != NULL) 225 mlx5_devx_qp_destroy(&qp->umr_qp_obj); 226 if (qp->qp_obj.qp != NULL) 227 mlx5_devx_qp_destroy(&qp->qp_obj); 228 if (qp->cq_obj.cq != NULL) 229 mlx5_devx_cq_destroy(&qp->cq_obj); 230 if (qp->mr.obj != NULL) { 231 void *opaq = qp->mr.addr; 232 233 priv->dereg_mr_cb(&qp->mr); 234 rte_free(opaq); 235 } 236 mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n); 237 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 238 rte_free(qp); 239 dev->data->queue_pairs[qp_id] = NULL; 240 return 0; 241 } 242 243 static void 244 mlx5_crypto_gcm_init_qp(struct mlx5_crypto_qp *qp) 245 { 246 volatile struct mlx5_gga_wqe *restrict wqe = 247 (volatile struct mlx5_gga_wqe *)qp->qp_obj.wqes; 248 volatile union mlx5_gga_crypto_opaque *opaq = qp->opaque_addr; 249 const uint32_t sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | 4u); 250 const uint32_t flags = RTE_BE32(MLX5_COMP_ALWAYS << 251 MLX5_COMP_MODE_OFFSET); 252 const uint32_t opaq_lkey = rte_cpu_to_be_32(qp->mr.lkey); 253 int i; 254 255 /* All the next fields state should stay constant. */ 256 for (i = 0; i < qp->entries_n; ++i, ++wqe) { 257 wqe->sq_ds = sq_ds; 258 wqe->flags = flags; 259 wqe->opaque_lkey = opaq_lkey; 260 wqe->opaque_vaddr = rte_cpu_to_be_64((uint64_t)(uintptr_t)&opaq[i]); 261 } 262 } 263 264 static inline int 265 mlx5_crypto_gcm_umr_qp_setup(struct rte_cryptodev *dev, struct mlx5_crypto_qp *qp, 266 int socket_id) 267 { 268 struct mlx5_crypto_priv *priv = dev->data->dev_private; 269 struct mlx5_devx_qp_attr attr = {0}; 270 uint32_t ret; 271 uint32_t log_wqbb_n; 272 273 /* Set UMR + SEND_EN WQE as maximum same with crypto. */ 274 log_wqbb_n = rte_log2_u32(qp->entries_n * 275 (MLX5_UMR_GCM_WQE_SET_SIZE / MLX5_SEND_WQE_BB)); 276 attr.pd = priv->cdev->pdn; 277 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj); 278 attr.cqn = qp->cq_obj.cq->id; 279 attr.num_of_receive_wqes = 0; 280 attr.num_of_send_wqbbs = RTE_BIT32(log_wqbb_n); 281 attr.ts_format = 282 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format); 283 attr.cd_master = 1; 284 ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->umr_qp_obj, 285 attr.num_of_send_wqbbs * MLX5_SEND_WQE_BB, 286 &attr, socket_id); 287 if (ret) { 288 DRV_LOG(ERR, "Failed to create UMR QP."); 289 return -1; 290 } 291 if (mlx5_devx_qp2rts(&qp->umr_qp_obj, qp->umr_qp_obj.qp->id)) { 292 DRV_LOG(ERR, "Failed to change UMR QP state to RTS."); 293 return -1; 294 } 295 /* Save the UMR WQEBBS for checking the WQE boundary. */ 296 qp->umr_wqbbs = attr.num_of_send_wqbbs; 297 return 0; 298 } 299 300 static int 301 mlx5_crypto_gcm_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, 302 const struct rte_cryptodev_qp_conf *qp_conf, 303 int socket_id) 304 { 305 struct mlx5_crypto_priv *priv = dev->data->dev_private; 306 struct mlx5_hca_attr *attr = &priv->cdev->config.hca_attr; 307 struct mlx5_crypto_qp *qp; 308 struct mlx5_devx_cq_attr cq_attr = { 309 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar.obj), 310 }; 311 struct mlx5_devx_qp_attr qp_attr = { 312 .pd = priv->cdev->pdn, 313 .uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj), 314 .user_index = qp_id, 315 }; 316 struct mlx5_devx_mkey_attr mkey_attr = { 317 .pd = priv->cdev->pdn, 318 .umr_en = 1, 319 .klm_num = priv->max_klm_num, 320 }; 321 uint32_t log_ops_n = rte_log2_u32(qp_conf->nb_descriptors); 322 uint32_t entries = RTE_BIT32(log_ops_n); 323 uint32_t alloc_size = sizeof(*qp); 324 size_t mr_size, opaq_size; 325 void *mr_buf; 326 int ret; 327 328 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 329 alloc_size += (sizeof(struct rte_crypto_op *) + 330 sizeof(struct mlx5_devx_obj *)) * entries; 331 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 332 socket_id); 333 if (qp == NULL) { 334 DRV_LOG(ERR, "Failed to allocate qp memory."); 335 rte_errno = ENOMEM; 336 return -rte_errno; 337 } 338 qp->priv = priv; 339 qp->entries_n = entries; 340 if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen, 341 priv->dev_config.socket_id)) { 342 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 343 (uint32_t)qp_id); 344 rte_errno = ENOMEM; 345 goto err; 346 } 347 /* 348 * The following KLM pointer must be aligned with 349 * MLX5_UMR_KLM_PTR_ALIGN. Aligned opaq_size here 350 * to make the KLM pointer with offset be aligned. 351 */ 352 opaq_size = RTE_ALIGN(sizeof(union mlx5_gga_crypto_opaque) * entries, 353 MLX5_UMR_KLM_PTR_ALIGN); 354 mr_size = (priv->max_klm_num * sizeof(struct mlx5_klm) * entries) + opaq_size; 355 mr_buf = rte_calloc(__func__, (size_t)1, mr_size, MLX5_UMR_KLM_PTR_ALIGN); 356 if (mr_buf == NULL) { 357 DRV_LOG(ERR, "Failed to allocate mr memory."); 358 rte_errno = ENOMEM; 359 goto err; 360 } 361 if (priv->reg_mr_cb(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) { 362 rte_free(mr_buf); 363 DRV_LOG(ERR, "Failed to register opaque MR."); 364 rte_errno = ENOMEM; 365 goto err; 366 } 367 qp->opaque_addr = qp->mr.addr; 368 qp->klm_array = RTE_PTR_ADD(qp->opaque_addr, opaq_size); 369 /* 370 * Triple the CQ size as UMR QP which contains UMR and SEND_EN WQE 371 * will share this CQ . 372 */ 373 qp->cq_entries_n = rte_align32pow2(entries * 3); 374 ret = mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, 375 rte_log2_u32(qp->cq_entries_n), 376 &cq_attr, socket_id); 377 if (ret != 0) { 378 DRV_LOG(ERR, "Failed to create CQ."); 379 goto err; 380 } 381 qp_attr.cqn = qp->cq_obj.cq->id; 382 qp_attr.ts_format = mlx5_ts_format_conv(attr->qp_ts_format); 383 qp_attr.num_of_receive_wqes = 0; 384 qp_attr.num_of_send_wqbbs = entries; 385 qp_attr.mmo = attr->crypto_mmo.crypto_mmo_qp; 386 /* Set MMO QP as follower as the input data may depend on UMR. */ 387 qp_attr.cd_slave_send = 1; 388 ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, 389 qp_attr.num_of_send_wqbbs * MLX5_WQE_SIZE, 390 &qp_attr, socket_id); 391 if (ret != 0) { 392 DRV_LOG(ERR, "Failed to create QP."); 393 goto err; 394 } 395 mlx5_crypto_gcm_init_qp(qp); 396 ret = mlx5_devx_qp2rts(&qp->qp_obj, 0); 397 if (ret) 398 goto err; 399 qp->ops = (struct rte_crypto_op **)(qp + 1); 400 qp->mkey = (struct mlx5_devx_obj **)(qp->ops + entries); 401 if (mlx5_crypto_gcm_umr_qp_setup(dev, qp, socket_id)) { 402 DRV_LOG(ERR, "Failed to setup UMR QP."); 403 goto err; 404 } 405 DRV_LOG(INFO, "QP %u: SQN=0x%X CQN=0x%X entries num = %u", 406 (uint32_t)qp_id, qp->qp_obj.qp->id, qp->cq_obj.cq->id, entries); 407 if (mlx5_crypto_indirect_mkeys_prepare(priv, qp, &mkey_attr, 408 mlx5_crypto_gcm_mkey_klm_update)) { 409 DRV_LOG(ERR, "Cannot allocate indirect memory regions."); 410 rte_errno = ENOMEM; 411 goto err; 412 } 413 dev->data->queue_pairs[qp_id] = qp; 414 return 0; 415 err: 416 mlx5_crypto_gcm_qp_release(dev, qp_id); 417 return -1; 418 } 419 420 static __rte_always_inline void 421 mlx5_crypto_gcm_get_op_info(struct mlx5_crypto_qp *qp, 422 struct rte_crypto_op *op, 423 struct mlx5_crypto_gcm_op_info *op_info) 424 { 425 struct mlx5_crypto_session *sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); 426 struct rte_mbuf *m_src = op->sym->m_src; 427 void *aad_addr = op->sym->aead.aad.data; 428 void *tag_addr = op->sym->aead.digest.data; 429 void *src_addr = rte_pktmbuf_mtod_offset(m_src, void *, op->sym->aead.data.offset); 430 struct rte_mbuf *m_dst = m_src; 431 void *dst_addr = src_addr; 432 void *expected_aad = NULL; 433 void *expected_tag = NULL; 434 bool is_enc = sess->op_type == MLX5_CRYPTO_OP_TYPE_ENCRYPTION; 435 bool cp_aad = false; 436 bool cp_tag = false; 437 438 op_info->is_oop = false; 439 op_info->need_umr = false; 440 op_info->is_enc = is_enc; 441 op_info->digest = NULL; 442 op_info->src_addr = aad_addr; 443 if (op->sym->m_dst && op->sym->m_dst != m_src) { 444 /* Add 2 for AAD and digest. */ 445 MLX5_ASSERT((uint32_t)(m_dst->nb_segs + m_src->nb_segs + 2) < 446 qp->priv->max_klm_num); 447 op_info->is_oop = true; 448 m_dst = op->sym->m_dst; 449 dst_addr = rte_pktmbuf_mtod_offset(m_dst, void *, op->sym->aead.data.offset); 450 if (m_dst->nb_segs > 1) { 451 op_info->need_umr = true; 452 return; 453 } 454 /* 455 * If the op's mbuf has extra data offset, don't copy AAD to 456 * this area. 457 */ 458 if (rte_pktmbuf_headroom(m_dst) < sess->aad_len || 459 op->sym->aead.data.offset) { 460 op_info->need_umr = true; 461 return; 462 } 463 } else { 464 /* Add 2 for AAD and digest. */ 465 MLX5_ASSERT((uint32_t)(m_src->nb_segs) + 2 < qp->priv->max_klm_num); 466 } 467 if (m_src->nb_segs > 1) { 468 op_info->need_umr = true; 469 return; 470 } 471 expected_aad = RTE_PTR_SUB(src_addr, sess->aad_len); 472 if (expected_aad != aad_addr) { 473 /* 474 * If the op's mbuf has extra data offset, don't copy AAD to 475 * this area. 476 */ 477 if (sess->aad_len > MLX5_CRYPTO_GCM_MAX_AAD || 478 sess->aad_len > rte_pktmbuf_headroom(m_src) || 479 op->sym->aead.data.offset) { 480 op_info->need_umr = true; 481 return; 482 } 483 cp_aad = true; 484 op_info->src_addr = expected_aad; 485 } 486 expected_tag = RTE_PTR_ADD(is_enc ? dst_addr : src_addr, op->sym->aead.data.length); 487 if (expected_tag != tag_addr) { 488 struct rte_mbuf *mbuf = is_enc ? m_dst : m_src; 489 490 /* 491 * If op's mbuf is not fully set as payload, don't copy digest to 492 * the left area. 493 */ 494 if (rte_pktmbuf_tailroom(mbuf) < sess->tag_len || 495 rte_pktmbuf_data_len(mbuf) != op->sym->aead.data.length) { 496 op_info->need_umr = true; 497 return; 498 } 499 if (is_enc) { 500 op_info->digest = expected_tag; 501 qp->cpy_tag_op++; 502 } else { 503 cp_tag = true; 504 } 505 } 506 if (cp_aad) 507 memcpy(expected_aad, aad_addr, sess->aad_len); 508 if (cp_tag) 509 memcpy(expected_tag, tag_addr, sess->tag_len); 510 } 511 512 static __rte_always_inline uint32_t 513 _mlx5_crypto_gcm_umr_build_mbuf_klm(struct mlx5_crypto_qp *qp, 514 struct rte_mbuf *mbuf, 515 struct mlx5_klm *klm, 516 uint32_t offset, 517 uint32_t *remain) 518 { 519 uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset); 520 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset); 521 522 if (data_len > *remain) 523 data_len = *remain; 524 *remain -= data_len; 525 klm->byte_count = rte_cpu_to_be_32(data_len); 526 klm->address = rte_cpu_to_be_64(addr); 527 klm->mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf); 528 return klm->mkey; 529 } 530 531 static __rte_always_inline int 532 mlx5_crypto_gcm_build_mbuf_chain_klms(struct mlx5_crypto_qp *qp, 533 struct rte_crypto_op *op, 534 struct rte_mbuf *mbuf, 535 struct mlx5_klm *klm) 536 { 537 uint32_t remain_len = op->sym->aead.data.length; 538 __rte_unused uint32_t nb_segs = mbuf->nb_segs; 539 uint32_t klm_n = 0; 540 541 /* mbuf seg num should be less than max_segs_num. */ 542 MLX5_ASSERT(nb_segs <= qp->priv->max_segs_num); 543 /* First mbuf needs to take the data offset. */ 544 if (unlikely(_mlx5_crypto_gcm_umr_build_mbuf_klm(qp, mbuf, klm, 545 op->sym->aead.data.offset, &remain_len) == UINT32_MAX)) { 546 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 547 return 0; 548 } 549 klm++; 550 klm_n++; 551 while (remain_len) { 552 nb_segs--; 553 mbuf = mbuf->next; 554 MLX5_ASSERT(mbuf && nb_segs); 555 if (unlikely(_mlx5_crypto_gcm_umr_build_mbuf_klm(qp, mbuf, klm, 556 0, &remain_len) == UINT32_MAX)) { 557 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 558 return 0; 559 } 560 klm++; 561 klm_n++; 562 } 563 return klm_n; 564 } 565 566 static __rte_always_inline int 567 mlx5_crypto_gcm_build_klm_by_addr(struct mlx5_crypto_qp *qp, 568 struct mlx5_klm *klm, 569 void *addr, 570 uint32_t len) 571 { 572 klm->byte_count = rte_cpu_to_be_32(len); 573 klm->address = rte_cpu_to_be_64((uintptr_t)addr); 574 klm->mkey = mlx5_mr_addr2mr_bh(&qp->mr_ctrl, (uintptr_t)addr); 575 if (klm->mkey == UINT32_MAX) 576 return 0; 577 return 1; 578 } 579 580 static __rte_always_inline int 581 mlx5_crypto_gcm_build_op_klm(struct mlx5_crypto_qp *qp, 582 struct rte_crypto_op *op, 583 struct mlx5_crypto_gcm_op_info *op_info, 584 struct mlx5_klm *klm, 585 uint32_t *len) 586 { 587 struct mlx5_crypto_session *sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); 588 struct mlx5_klm *digest = NULL, *aad = NULL; 589 uint32_t total_len = op->sym->aead.data.length + sess->aad_len + sess->tag_len; 590 uint32_t klm_n = 0, klm_src = 0, klm_dst = 0; 591 592 /* Build AAD KLM. */ 593 aad = klm; 594 if (!mlx5_crypto_gcm_build_klm_by_addr(qp, aad, op->sym->aead.aad.data, sess->aad_len)) 595 return 0; 596 klm_n++; 597 /* Build src mubf KLM. */ 598 klm_src = mlx5_crypto_gcm_build_mbuf_chain_klms(qp, op, op->sym->m_src, &klm[klm_n]); 599 if (!klm_src) 600 return 0; 601 klm_n += klm_src; 602 /* Reserve digest KLM if needed. */ 603 if (!op_info->is_oop || 604 sess->op_type == MLX5_CRYPTO_OP_TYPE_DECRYPTION) { 605 digest = &klm[klm_n]; 606 klm_n++; 607 } 608 /* Build dst mbuf KLM. */ 609 if (op_info->is_oop) { 610 klm[klm_n] = *aad; 611 klm_n++; 612 klm_dst = mlx5_crypto_gcm_build_mbuf_chain_klms(qp, op, op->sym->m_dst, 613 &klm[klm_n]); 614 if (!klm_dst) 615 return 0; 616 klm_n += klm_dst; 617 total_len += (op->sym->aead.data.length + sess->aad_len); 618 } 619 /* Update digest at the end if it is not set. */ 620 if (!digest) { 621 digest = &klm[klm_n]; 622 klm_n++; 623 } 624 /* Build digest KLM. */ 625 if (!mlx5_crypto_gcm_build_klm_by_addr(qp, digest, op->sym->aead.digest.data, 626 sess->tag_len)) 627 return 0; 628 *len = total_len; 629 return klm_n; 630 } 631 632 static __rte_always_inline struct mlx5_wqe_cseg * 633 mlx5_crypto_gcm_get_umr_wqe(struct mlx5_crypto_qp *qp) 634 { 635 uint32_t wqe_offset = qp->umr_pi & (qp->umr_wqbbs - 1); 636 uint32_t left_wqbbs = qp->umr_wqbbs - wqe_offset; 637 struct mlx5_wqe_cseg *wqe; 638 639 /* If UMR WQE is near the boundary. */ 640 if (left_wqbbs < MLX5_UMR_GCM_WQE_STRIDE) { 641 /* Append NOP WQE as the left WQEBBS is not enough for UMR. */ 642 wqe = RTE_PTR_ADD(qp->umr_qp_obj.umem_buf, wqe_offset * MLX5_SEND_WQE_BB); 643 wqe->opcode = rte_cpu_to_be_32(MLX5_OPCODE_NOP | ((uint32_t)qp->umr_pi << 8)); 644 wqe->sq_ds = rte_cpu_to_be_32((qp->umr_qp_obj.qp->id << 8) | (left_wqbbs << 2)); 645 wqe->flags = RTE_BE32(0); 646 wqe->misc = RTE_BE32(0); 647 qp->umr_pi += left_wqbbs; 648 wqe_offset = qp->umr_pi & (qp->umr_wqbbs - 1); 649 } 650 wqe_offset *= MLX5_SEND_WQE_BB; 651 return RTE_PTR_ADD(qp->umr_qp_obj.umem_buf, wqe_offset); 652 } 653 654 static __rte_always_inline int 655 mlx5_crypto_gcm_build_umr(struct mlx5_crypto_qp *qp, 656 struct rte_crypto_op *op, 657 uint32_t idx, 658 struct mlx5_crypto_gcm_op_info *op_info, 659 struct mlx5_crypto_gcm_data *data) 660 { 661 struct mlx5_crypto_priv *priv = qp->priv; 662 struct mlx5_crypto_session *sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); 663 struct mlx5_wqe_cseg *wqe; 664 struct mlx5_wqe_umr_cseg *ucseg; 665 struct mlx5_wqe_mkey_cseg *mkc; 666 struct mlx5_klm *iklm; 667 struct mlx5_klm *klm = &qp->klm_array[idx * priv->max_klm_num]; 668 uint16_t klm_size, klm_align; 669 uint32_t total_len; 670 671 /* Build KLM base on the op. */ 672 klm_size = mlx5_crypto_gcm_build_op_klm(qp, op, op_info, klm, &total_len); 673 if (!klm_size) 674 return -EINVAL; 675 klm_align = RTE_ALIGN(klm_size, 4); 676 /* Get UMR WQE memory. */ 677 wqe = mlx5_crypto_gcm_get_umr_wqe(qp); 678 memset(wqe, 0, MLX5_UMR_GCM_WQE_SIZE); 679 /* Set WQE control seg. Non-inline KLM UMR WQE size must be 9 WQE_DS. */ 680 wqe->opcode = rte_cpu_to_be_32(MLX5_OPCODE_UMR | ((uint32_t)qp->umr_pi << 8)); 681 wqe->sq_ds = rte_cpu_to_be_32((qp->umr_qp_obj.qp->id << 8) | 9); 682 wqe->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << MLX5_COMP_MODE_OFFSET); 683 wqe->misc = rte_cpu_to_be_32(qp->mkey[idx]->id); 684 /* Set UMR WQE control seg. */ 685 ucseg = (struct mlx5_wqe_umr_cseg *)(wqe + 1); 686 ucseg->mkey_mask |= RTE_BE64(1u << 0); 687 ucseg->ko_to_bs = rte_cpu_to_be_32(klm_align << MLX5_UMRC_KO_OFFSET); 688 /* Set mkey context seg. */ 689 mkc = (struct mlx5_wqe_mkey_cseg *)(ucseg + 1); 690 mkc->len = rte_cpu_to_be_64(total_len); 691 mkc->qpn_mkey = rte_cpu_to_be_32(0xffffff00 | (qp->mkey[idx]->id & 0xff)); 692 /* Set UMR pointer to data seg. */ 693 iklm = (struct mlx5_klm *)(mkc + 1); 694 iklm->address = rte_cpu_to_be_64((uintptr_t)((char *)klm)); 695 iklm->mkey = rte_cpu_to_be_32(qp->mr.lkey); 696 data->src_mkey = rte_cpu_to_be_32(qp->mkey[idx]->id); 697 data->dst_mkey = data->src_mkey; 698 data->src_addr = 0; 699 data->src_bytes = sess->aad_len + op->sym->aead.data.length; 700 data->dst_bytes = data->src_bytes; 701 if (op_info->is_enc) 702 data->dst_bytes += sess->tag_len; 703 else 704 data->src_bytes += sess->tag_len; 705 if (op_info->is_oop) 706 data->dst_addr = (void *)(uintptr_t)(data->src_bytes); 707 else 708 data->dst_addr = 0; 709 /* Clear the padding memory. */ 710 memset(&klm[klm_size], 0, sizeof(struct mlx5_klm) * (klm_align - klm_size)); 711 /* Update PI and WQE */ 712 qp->umr_pi += MLX5_UMR_GCM_WQE_STRIDE; 713 qp->umr_wqe = (uint8_t *)wqe; 714 return 0; 715 } 716 717 static __rte_always_inline void 718 mlx5_crypto_gcm_build_send_en(struct mlx5_crypto_qp *qp) 719 { 720 uint32_t wqe_offset = (qp->umr_pi & (qp->umr_wqbbs - 1)) * MLX5_SEND_WQE_BB; 721 struct mlx5_wqe_cseg *cs = RTE_PTR_ADD(qp->umr_qp_obj.wqes, wqe_offset); 722 struct mlx5_wqe_qseg *qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 723 724 cs->opcode = rte_cpu_to_be_32(MLX5_OPCODE_SEND_EN | ((uint32_t)qp->umr_pi << 8)); 725 cs->sq_ds = rte_cpu_to_be_32((qp->umr_qp_obj.qp->id << 8) | 2); 726 /* 727 * No need to generate the SEND_EN CQE as we want only GGA CQE 728 * in the CQ normally. We can compare qp->last_send_gga_pi with 729 * qp->pi to know if all SEND_EN be consumed. 730 */ 731 cs->flags = RTE_BE32((MLX5_COMP_ONLY_FIRST_ERR << MLX5_COMP_MODE_OFFSET) | 732 MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE); 733 cs->misc = RTE_BE32(0); 734 qs->max_index = rte_cpu_to_be_32(qp->pi); 735 qs->qpn_cqn = rte_cpu_to_be_32(qp->qp_obj.qp->id); 736 qp->umr_wqe = (uint8_t *)cs; 737 qp->umr_pi += 1; 738 } 739 740 static __rte_always_inline void 741 mlx5_crypto_gcm_wqe_set(struct mlx5_crypto_qp *qp, 742 struct rte_crypto_op *op, 743 uint32_t idx, 744 struct mlx5_crypto_gcm_data *data) 745 { 746 struct mlx5_crypto_session *sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); 747 struct mlx5_gga_wqe *wqe = &((struct mlx5_gga_wqe *)qp->qp_obj.wqes)[idx]; 748 union mlx5_gga_crypto_opaque *opaq = qp->opaque_addr; 749 750 memcpy(opaq[idx].cp.iv, 751 rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), sess->iv_len); 752 opaq[idx].cp.tag_size = sess->wqe_tag_len; 753 opaq[idx].cp.aad_size = sess->wqe_aad_len; 754 /* Update control seg. */ 755 wqe->opcode = rte_cpu_to_be_32(MLX5_MMO_CRYPTO_OPC + (qp->pi << 8)); 756 wqe->gga_ctrl1 = sess->mmo_ctrl; 757 wqe->gga_ctrl2 = sess->dek_id; 758 wqe->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << MLX5_COMP_MODE_OFFSET); 759 /* Update op_info seg. */ 760 wqe->gather.bcount = rte_cpu_to_be_32(data->src_bytes); 761 wqe->gather.lkey = data->src_mkey; 762 wqe->gather.pbuf = rte_cpu_to_be_64((uintptr_t)data->src_addr); 763 /* Update output seg. */ 764 wqe->scatter.bcount = rte_cpu_to_be_32(data->dst_bytes); 765 wqe->scatter.lkey = data->dst_mkey; 766 wqe->scatter.pbuf = rte_cpu_to_be_64((uintptr_t)data->dst_addr); 767 qp->wqe = (uint8_t *)wqe; 768 } 769 770 static uint16_t 771 mlx5_crypto_gcm_enqueue_burst(void *queue_pair, 772 struct rte_crypto_op **ops, 773 uint16_t nb_ops) 774 { 775 struct mlx5_crypto_qp *qp = queue_pair; 776 struct mlx5_crypto_session *sess; 777 struct mlx5_crypto_priv *priv = qp->priv; 778 struct mlx5_crypto_gcm_tag_cpy_info *tag; 779 struct mlx5_crypto_gcm_data gcm_data; 780 struct rte_crypto_op *op; 781 struct mlx5_crypto_gcm_op_info op_info; 782 uint16_t mask = qp->entries_n - 1; 783 uint16_t remain = qp->entries_n - (qp->pi - qp->qp_ci); 784 uint32_t idx; 785 uint16_t umr_cnt = 0; 786 787 if (remain < nb_ops) 788 nb_ops = remain; 789 else 790 remain = nb_ops; 791 if (unlikely(remain == 0)) 792 return 0; 793 do { 794 op = *ops++; 795 sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); 796 idx = qp->pi & mask; 797 mlx5_crypto_gcm_get_op_info(qp, op, &op_info); 798 if (!op_info.need_umr) { 799 gcm_data.src_addr = op_info.src_addr; 800 gcm_data.src_bytes = op->sym->aead.data.length + sess->aad_len; 801 gcm_data.src_mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, op->sym->m_src); 802 if (op_info.is_oop) { 803 gcm_data.dst_addr = RTE_PTR_SUB 804 (rte_pktmbuf_mtod_offset(op->sym->m_dst, 805 void *, op->sym->aead.data.offset), sess->aad_len); 806 gcm_data.dst_mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, op->sym->m_dst); 807 } else { 808 gcm_data.dst_addr = gcm_data.src_addr; 809 gcm_data.dst_mkey = gcm_data.src_mkey; 810 } 811 gcm_data.dst_bytes = gcm_data.src_bytes; 812 if (op_info.is_enc) 813 gcm_data.dst_bytes += sess->tag_len; 814 else 815 gcm_data.src_bytes += sess->tag_len; 816 } else { 817 if (unlikely(mlx5_crypto_gcm_build_umr(qp, op, idx, 818 &op_info, &gcm_data))) { 819 qp->stats.enqueue_err_count++; 820 if (remain != nb_ops) { 821 qp->stats.enqueued_count -= remain; 822 break; 823 } 824 return 0; 825 } 826 umr_cnt++; 827 } 828 mlx5_crypto_gcm_wqe_set(qp, op, idx, &gcm_data); 829 if (op_info.digest) { 830 tag = (struct mlx5_crypto_gcm_tag_cpy_info *)op->sym->aead.digest.data; 831 tag->digest = op_info.digest; 832 tag->tag_len = sess->tag_len; 833 op->status = MLX5_CRYPTO_OP_STATUS_GCM_TAG_COPY; 834 } else { 835 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 836 } 837 qp->ops[idx] = op; 838 qp->pi++; 839 } while (--remain); 840 qp->stats.enqueued_count += nb_ops; 841 /* Update the last GGA cseg with COMP. */ 842 ((struct mlx5_wqe_cseg *)qp->wqe)->flags = 843 RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET); 844 /* Only when there are no pending SEND_EN WQEs in background. */ 845 if (!umr_cnt && !qp->has_umr) { 846 mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->wqe, 847 qp->pi, &qp->qp_obj.db_rec[MLX5_SND_DBR], 848 !priv->uar.dbnc); 849 } else { 850 mlx5_crypto_gcm_build_send_en(qp); 851 mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->umr_wqe, 852 qp->umr_pi, &qp->umr_qp_obj.db_rec[MLX5_SND_DBR], 853 !priv->uar.dbnc); 854 qp->last_gga_pi = qp->pi; 855 qp->has_umr = true; 856 } 857 return nb_ops; 858 } 859 860 static __rte_noinline void 861 mlx5_crypto_gcm_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op) 862 { 863 uint8_t op_code; 864 const uint32_t idx = qp->cq_ci & (qp->entries_n - 1); 865 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *) 866 &qp->cq_obj.cqes[idx]; 867 868 op_code = rte_be_to_cpu_32(cqe->s_wqe_opcode_qpn) >> MLX5_CQ_INDEX_WIDTH; 869 DRV_LOG(ERR, "CQE ERR:0x%x, Vendor_ERR:0x%x, OP:0x%x, QPN:0x%x, WQE_CNT:0x%x", 870 cqe->syndrome, cqe->vendor_err_synd, op_code, 871 (rte_be_to_cpu_32(cqe->s_wqe_opcode_qpn) & 0xffffff), 872 rte_be_to_cpu_16(cqe->wqe_counter)); 873 if (op && op_code == MLX5_OPCODE_MMO) { 874 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 875 qp->stats.dequeue_err_count++; 876 } 877 } 878 879 static __rte_always_inline void 880 mlx5_crypto_gcm_fill_op(struct mlx5_crypto_qp *qp, 881 struct rte_crypto_op **ops, 882 uint16_t orci, 883 uint16_t rci, 884 uint16_t op_mask) 885 { 886 uint16_t n; 887 888 orci &= op_mask; 889 rci &= op_mask; 890 if (unlikely(orci > rci)) { 891 n = op_mask - orci + 1; 892 memcpy(ops, &qp->ops[orci], n * sizeof(*ops)); 893 orci = 0; 894 } else { 895 n = 0; 896 } 897 /* rci can be 0 here, memcpy will skip that. */ 898 memcpy(&ops[n], &qp->ops[orci], (rci - orci) * sizeof(*ops)); 899 } 900 901 static __rte_always_inline void 902 mlx5_crypto_gcm_cpy_tag(struct mlx5_crypto_qp *qp, 903 uint16_t orci, 904 uint16_t rci, 905 uint16_t op_mask) 906 { 907 struct rte_crypto_op *op; 908 struct mlx5_crypto_gcm_tag_cpy_info *tag; 909 910 while (qp->cpy_tag_op && orci != rci) { 911 op = qp->ops[orci & op_mask]; 912 if (op->status == MLX5_CRYPTO_OP_STATUS_GCM_TAG_COPY) { 913 tag = (struct mlx5_crypto_gcm_tag_cpy_info *)op->sym->aead.digest.data; 914 memcpy(op->sym->aead.digest.data, tag->digest, tag->tag_len); 915 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 916 qp->cpy_tag_op--; 917 } 918 orci++; 919 } 920 } 921 922 static uint16_t 923 mlx5_crypto_gcm_dequeue_burst(void *queue_pair, 924 struct rte_crypto_op **ops, 925 uint16_t nb_ops) 926 { 927 struct mlx5_crypto_qp *qp = queue_pair; 928 volatile struct mlx5_cqe *restrict cqe; 929 const unsigned int cq_size = qp->cq_entries_n; 930 const unsigned int mask = cq_size - 1; 931 const unsigned int op_mask = qp->entries_n - 1; 932 uint32_t idx; 933 uint32_t next_idx = qp->cq_ci & mask; 934 uint16_t reported_ci = qp->reported_ci; 935 uint16_t qp_ci = qp->qp_ci; 936 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - reported_ci), nb_ops); 937 uint16_t op_num = 0; 938 int ret; 939 940 if (unlikely(max == 0)) 941 return 0; 942 while (qp_ci - reported_ci < max) { 943 idx = next_idx; 944 next_idx = (qp->cq_ci + 1) & mask; 945 cqe = &qp->cq_obj.cqes[idx]; 946 ret = check_cqe(cqe, cq_size, qp->cq_ci); 947 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 948 if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN)) 949 mlx5_crypto_gcm_cqe_err_handle(qp, 950 qp->ops[reported_ci & op_mask]); 951 break; 952 } 953 qp_ci = rte_be_to_cpu_16(cqe->wqe_counter) + 1; 954 if (qp->has_umr && 955 (qp->last_gga_pi + 1) == qp_ci) 956 qp->has_umr = false; 957 qp->cq_ci++; 958 } 959 /* If wqe_counter changed, means CQE handled. */ 960 if (likely(qp->qp_ci != qp_ci)) { 961 qp->qp_ci = qp_ci; 962 rte_io_wmb(); 963 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->cq_ci); 964 } 965 /* If reported_ci is not same with qp_ci, means op retrieved. */ 966 if (qp_ci != reported_ci) { 967 op_num = RTE_MIN((uint16_t)(qp_ci - reported_ci), max); 968 reported_ci += op_num; 969 mlx5_crypto_gcm_cpy_tag(qp, qp->reported_ci, reported_ci, op_mask); 970 mlx5_crypto_gcm_fill_op(qp, ops, qp->reported_ci, reported_ci, op_mask); 971 qp->stats.dequeued_count += op_num; 972 qp->reported_ci = reported_ci; 973 } 974 return op_num; 975 } 976 977 int 978 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv) 979 { 980 struct mlx5_common_device *cdev = priv->cdev; 981 struct rte_cryptodev *crypto_dev = priv->crypto_dev; 982 struct rte_cryptodev_ops *dev_ops = crypto_dev->dev_ops; 983 int ret; 984 985 /* Override AES-GCM specified ops. */ 986 dev_ops->sym_session_configure = mlx5_crypto_sym_gcm_session_configure; 987 mlx5_os_set_reg_mr_cb(&priv->reg_mr_cb, &priv->dereg_mr_cb); 988 dev_ops->queue_pair_setup = mlx5_crypto_gcm_qp_setup; 989 dev_ops->queue_pair_release = mlx5_crypto_gcm_qp_release; 990 crypto_dev->dequeue_burst = mlx5_crypto_gcm_dequeue_burst; 991 crypto_dev->enqueue_burst = mlx5_crypto_gcm_enqueue_burst; 992 priv->max_klm_num = RTE_ALIGN((priv->max_segs_num + 1) * 2 + 1, MLX5_UMR_KLM_NUM_ALIGN); 993 /* Generate GCM capability. */ 994 ret = mlx5_crypto_generate_gcm_cap(&cdev->config.hca_attr.crypto_mmo, 995 mlx5_crypto_gcm_caps); 996 if (ret) { 997 DRV_LOG(ERR, "No enough AES-GCM cap."); 998 return -1; 999 } 1000 priv->caps = mlx5_crypto_gcm_caps; 1001 return 0; 1002 } 1003