1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_errno.h> 8 #include <rte_log.h> 9 #include <rte_bus_pci.h> 10 #include <rte_memory.h> 11 12 #include <mlx5_glue.h> 13 #include <mlx5_common.h> 14 #include <mlx5_devx_cmds.h> 15 #include <mlx5_common_os.h> 16 17 #include "mlx5_crypto_utils.h" 18 #include "mlx5_crypto.h" 19 20 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 21 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 22 #define MLX5_CRYPTO_MAX_QPS 1024 23 #define MLX5_CRYPTO_MAX_SEGS 56 24 25 #define MLX5_CRYPTO_FEATURE_FLAGS \ 26 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 27 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \ 28 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \ 29 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \ 30 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \ 31 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \ 32 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 33 34 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 35 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 36 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER; 37 38 int mlx5_crypto_logtype; 39 40 uint8_t mlx5_crypto_driver_id; 41 42 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = { 43 { /* AES XTS */ 44 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, 45 {.sym = { 46 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, 47 {.cipher = { 48 .algo = RTE_CRYPTO_CIPHER_AES_XTS, 49 .block_size = 16, 50 .key_size = { 51 .min = 32, 52 .max = 64, 53 .increment = 32 54 }, 55 .iv_size = { 56 .min = 16, 57 .max = 16, 58 .increment = 0 59 }, 60 .dataunit_set = 61 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES | 62 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES, 63 }, } 64 }, } 65 }, 66 }; 67 68 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 69 70 static const struct rte_driver mlx5_drv = { 71 .name = mlx5_crypto_drv_name, 72 .alias = mlx5_crypto_drv_name 73 }; 74 75 static struct cryptodev_driver mlx5_cryptodev_driver; 76 77 struct mlx5_crypto_session { 78 uint32_t bs_bpt_eo_es; 79 /**< bsf_size, bsf_p_type, encryption_order and encryption standard, 80 * saved in big endian format. 81 */ 82 uint32_t bsp_res; 83 /**< crypto_block_size_pointer and reserved 24 bits saved in big 84 * endian format. 85 */ 86 uint32_t iv_offset:16; 87 /**< Starting point for Initialisation Vector. */ 88 struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */ 89 uint32_t dek_id; /**< DEK ID */ 90 } __rte_packed; 91 92 static void 93 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 94 struct rte_cryptodev_info *dev_info) 95 { 96 RTE_SET_USED(dev); 97 if (dev_info != NULL) { 98 dev_info->driver_id = mlx5_crypto_driver_id; 99 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 100 dev_info->capabilities = mlx5_crypto_caps; 101 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 102 dev_info->min_mbuf_headroom_req = 0; 103 dev_info->min_mbuf_tailroom_req = 0; 104 dev_info->sym.max_nb_sessions = 0; 105 /* 106 * If 0, the device does not have any limitation in number of 107 * sessions that can be used. 108 */ 109 } 110 } 111 112 static int 113 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 114 struct rte_cryptodev_config *config) 115 { 116 struct mlx5_crypto_priv *priv = dev->data->dev_private; 117 118 if (config == NULL) { 119 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 120 return -EINVAL; 121 } 122 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 123 DRV_LOG(ERR, 124 "Disabled symmetric crypto feature is not supported."); 125 return -ENOTSUP; 126 } 127 if (mlx5_crypto_dek_setup(priv) != 0) { 128 DRV_LOG(ERR, "Dek hash list creation has failed."); 129 return -ENOMEM; 130 } 131 priv->dev_config = *config; 132 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 133 return 0; 134 } 135 136 static void 137 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 138 { 139 RTE_SET_USED(dev); 140 } 141 142 static int 143 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 144 { 145 RTE_SET_USED(dev); 146 return 0; 147 } 148 149 static int 150 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 151 { 152 struct mlx5_crypto_priv *priv = dev->data->dev_private; 153 154 mlx5_crypto_dek_unset(priv); 155 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 156 return 0; 157 } 158 159 static unsigned int 160 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 161 { 162 return sizeof(struct mlx5_crypto_session); 163 } 164 165 static int 166 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev, 167 struct rte_crypto_sym_xform *xform, 168 struct rte_cryptodev_sym_session *session, 169 struct rte_mempool *mp) 170 { 171 struct mlx5_crypto_priv *priv = dev->data->dev_private; 172 struct mlx5_crypto_session *sess_private_data; 173 struct rte_crypto_cipher_xform *cipher; 174 uint8_t encryption_order; 175 int ret; 176 177 if (unlikely(xform->next != NULL)) { 178 DRV_LOG(ERR, "Xform next is not supported."); 179 return -ENOTSUP; 180 } 181 if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) || 182 (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) { 183 DRV_LOG(ERR, "Only AES-XTS algorithm is supported."); 184 return -ENOTSUP; 185 } 186 ret = rte_mempool_get(mp, (void *)&sess_private_data); 187 if (ret != 0) { 188 DRV_LOG(ERR, 189 "Failed to get session %p private data from mempool.", 190 sess_private_data); 191 return -ENOMEM; 192 } 193 cipher = &xform->cipher; 194 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher); 195 if (sess_private_data->dek == NULL) { 196 rte_mempool_put(mp, sess_private_data); 197 DRV_LOG(ERR, "Failed to prepare dek."); 198 return -ENOMEM; 199 } 200 if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 201 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY; 202 else 203 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE; 204 sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32 205 (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET | 206 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET | 207 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET | 208 MLX5_ENCRYPTION_STANDARD_AES_XTS); 209 switch (xform->cipher.dataunit_len) { 210 case 0: 211 sess_private_data->bsp_res = 0; 212 break; 213 case 512: 214 sess_private_data->bsp_res = rte_cpu_to_be_32 215 ((uint32_t)MLX5_BLOCK_SIZE_512B << 216 MLX5_BLOCK_SIZE_OFFSET); 217 break; 218 case 4096: 219 sess_private_data->bsp_res = rte_cpu_to_be_32 220 ((uint32_t)MLX5_BLOCK_SIZE_4096B << 221 MLX5_BLOCK_SIZE_OFFSET); 222 break; 223 default: 224 DRV_LOG(ERR, "Cipher data unit length is not supported."); 225 return -ENOTSUP; 226 } 227 sess_private_data->iv_offset = cipher->iv.offset; 228 sess_private_data->dek_id = 229 rte_cpu_to_be_32(sess_private_data->dek->obj->id & 230 0xffffff); 231 set_sym_session_private_data(session, dev->driver_id, 232 sess_private_data); 233 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data); 234 return 0; 235 } 236 237 static void 238 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 239 struct rte_cryptodev_sym_session *sess) 240 { 241 struct mlx5_crypto_priv *priv = dev->data->dev_private; 242 struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess, 243 dev->driver_id); 244 245 if (unlikely(spriv == NULL)) { 246 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 247 return; 248 } 249 mlx5_crypto_dek_destroy(priv, spriv->dek); 250 set_sym_session_private_data(sess, dev->driver_id, NULL); 251 rte_mempool_put(rte_mempool_from_obj(spriv), spriv); 252 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 253 } 254 255 static void 256 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n) 257 { 258 uint16_t i; 259 260 for (i = 0; i < n; i++) 261 if (qp->mkey[i]) 262 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i])); 263 } 264 265 static void 266 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp) 267 { 268 if (qp == NULL) 269 return; 270 if (qp->qp_obj != NULL) 271 claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj)); 272 if (qp->umem_obj != NULL) 273 claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj)); 274 if (qp->umem_buf != NULL) 275 rte_free(qp->umem_buf); 276 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 277 mlx5_devx_cq_destroy(&qp->cq_obj); 278 rte_free(qp); 279 } 280 281 static int 282 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id) 283 { 284 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 285 286 mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n); 287 mlx5_crypto_qp_release(qp); 288 dev->data->queue_pairs[qp_id] = NULL; 289 return 0; 290 } 291 292 static int 293 mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp) 294 { 295 /* 296 * In Order to configure self loopback, when calling these functions the 297 * remote QP id that is used is the id of the same QP. 298 */ 299 if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP, 300 qp->qp_obj->id)) { 301 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).", 302 rte_errno); 303 return -1; 304 } 305 if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP, 306 qp->qp_obj->id)) { 307 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).", 308 rte_errno); 309 return -1; 310 } 311 if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP, 312 qp->qp_obj->id)) { 313 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).", 314 rte_errno); 315 return -1; 316 } 317 return 0; 318 } 319 320 static __rte_noinline uint32_t 321 mlx5_crypto_get_block_size(struct rte_crypto_op *op) 322 { 323 uint32_t bl = op->sym->cipher.data.length; 324 325 switch (bl) { 326 case (1 << 20): 327 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET); 328 case (1 << 12): 329 return RTE_BE32(MLX5_BLOCK_SIZE_4096B << 330 MLX5_BLOCK_SIZE_OFFSET); 331 case (1 << 9): 332 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET); 333 default: 334 DRV_LOG(ERR, "Unknown block size: %u.", bl); 335 return UINT32_MAX; 336 } 337 } 338 339 /** 340 * Query LKey from a packet buffer for QP. If not found, add the mempool. 341 * 342 * @param priv 343 * Pointer to the priv object. 344 * @param addr 345 * Search key. 346 * @param mr_ctrl 347 * Pointer to per-queue MR control structure. 348 * @param ol_flags 349 * Mbuf offload features. 350 * 351 * @return 352 * Searched LKey on success, UINT32_MAX on no match. 353 */ 354 static __rte_always_inline uint32_t 355 mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr, 356 struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags) 357 { 358 uint32_t lkey; 359 360 /* Check generation bit to see if there's any change on existing MRs. */ 361 if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen)) 362 mlx5_mr_flush_local_cache(mr_ctrl); 363 /* Linear search on MR cache array. */ 364 lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru, 365 MLX5_MR_CACHE_N, addr); 366 if (likely(lkey != UINT32_MAX)) 367 return lkey; 368 /* Take slower bottom-half on miss. */ 369 return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr, 370 !!(ol_flags & EXT_ATTACHED_MBUF)); 371 } 372 373 static __rte_always_inline uint32_t 374 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp, 375 struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm, 376 uint32_t offset, uint32_t *remain) 377 { 378 uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset); 379 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset); 380 381 if (data_len > *remain) 382 data_len = *remain; 383 *remain -= data_len; 384 klm->bcount = rte_cpu_to_be_32(data_len); 385 klm->pbuf = rte_cpu_to_be_64(addr); 386 klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl, 387 mbuf->ol_flags); 388 return klm->lkey; 389 390 } 391 392 static __rte_always_inline uint32_t 393 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp, 394 struct rte_crypto_op *op, struct rte_mbuf *mbuf, 395 struct mlx5_wqe_dseg *klm) 396 { 397 uint32_t remain_len = op->sym->cipher.data.length; 398 uint32_t nb_segs = mbuf->nb_segs; 399 uint32_t klm_n = 1u; 400 401 /* First mbuf needs to take the cipher offset. */ 402 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm, 403 op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) { 404 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 405 return 0; 406 } 407 while (remain_len) { 408 nb_segs--; 409 mbuf = mbuf->next; 410 if (unlikely(mbuf == NULL || nb_segs == 0)) { 411 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 412 return 0; 413 } 414 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0, 415 &remain_len) == UINT32_MAX)) { 416 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 417 return 0; 418 } 419 klm_n++; 420 } 421 return klm_n; 422 } 423 424 static __rte_always_inline int 425 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv, 426 struct mlx5_crypto_qp *qp, 427 struct rte_crypto_op *op, 428 struct mlx5_umr_wqe *umr) 429 { 430 struct mlx5_crypto_session *sess = get_sym_session_private_data 431 (op->sym->session, mlx5_crypto_driver_id); 432 struct mlx5_wqe_cseg *cseg = &umr->ctr; 433 struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc; 434 struct mlx5_wqe_dseg *klms = &umr->kseg[0]; 435 struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *) 436 RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1; 437 uint32_t ds; 438 bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src; 439 /* Set UMR WQE. */ 440 uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op, 441 ipl ? op->sym->m_src : op->sym->m_dst, klms); 442 443 if (unlikely(klm_n == 0)) 444 return 0; 445 bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es; 446 if (unlikely(!sess->bsp_res)) { 447 bsf->bsp_res = mlx5_crypto_get_block_size(op); 448 if (unlikely(bsf->bsp_res == UINT32_MAX)) { 449 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 450 return 0; 451 } 452 } else { 453 bsf->bsp_res = sess->bsp_res; 454 } 455 bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length); 456 memcpy(bsf->xts_initial_tweak, 457 rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16); 458 bsf->res_dp = sess->dek_id; 459 mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length); 460 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR); 461 qp->db_pi += priv->umr_wqe_stride; 462 /* Set RDMA_WRITE WQE. */ 463 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 464 klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe)); 465 if (!ipl) { 466 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src, 467 klms); 468 if (unlikely(klm_n == 0)) 469 return 0; 470 } else { 471 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n); 472 } 473 ds = 2 + klm_n; 474 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds); 475 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 476 MLX5_OPCODE_RDMA_WRITE); 477 ds = RTE_ALIGN(ds, 4); 478 qp->db_pi += ds >> 2; 479 /* Set NOP WQE if needed. */ 480 if (priv->max_rdmar_ds > ds) { 481 cseg += ds; 482 ds = priv->max_rdmar_ds - ds; 483 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds); 484 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 485 MLX5_OPCODE_NOP); 486 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */ 487 } 488 qp->wqe = (uint8_t *)cseg; 489 return 1; 490 } 491 492 static __rte_always_inline void 493 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv) 494 { 495 #ifdef RTE_ARCH_64 496 *priv->uar_addr = val; 497 #else /* !RTE_ARCH_64 */ 498 rte_spinlock_lock(&priv->uar32_sl); 499 *(volatile uint32_t *)priv->uar_addr = val; 500 rte_io_wmb(); 501 *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32; 502 rte_spinlock_unlock(&priv->uar32_sl); 503 #endif 504 } 505 506 static uint16_t 507 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 508 uint16_t nb_ops) 509 { 510 struct mlx5_crypto_qp *qp = queue_pair; 511 struct mlx5_crypto_priv *priv = qp->priv; 512 struct mlx5_umr_wqe *umr; 513 struct rte_crypto_op *op; 514 uint16_t mask = qp->entries_n - 1; 515 uint16_t remain = qp->entries_n - (qp->pi - qp->ci); 516 uint32_t idx; 517 518 if (remain < nb_ops) 519 nb_ops = remain; 520 else 521 remain = nb_ops; 522 if (unlikely(remain == 0)) 523 return 0; 524 do { 525 idx = qp->pi & mask; 526 op = *ops++; 527 umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * idx); 528 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) { 529 qp->stats.enqueue_err_count++; 530 if (remain != nb_ops) { 531 qp->stats.enqueued_count -= remain; 532 break; 533 } 534 return 0; 535 } 536 qp->ops[idx] = op; 537 qp->pi++; 538 } while (--remain); 539 qp->stats.enqueued_count += nb_ops; 540 rte_io_wmb(); 541 qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi); 542 rte_wmb(); 543 mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv); 544 rte_wmb(); 545 return nb_ops; 546 } 547 548 static __rte_noinline void 549 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op) 550 { 551 const uint32_t idx = qp->ci & (qp->entries_n - 1); 552 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *) 553 &qp->cq_obj.cqes[idx]; 554 555 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 556 qp->stats.dequeue_err_count++; 557 DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome)); 558 } 559 560 static uint16_t 561 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 562 uint16_t nb_ops) 563 { 564 struct mlx5_crypto_qp *qp = queue_pair; 565 volatile struct mlx5_cqe *restrict cqe; 566 struct rte_crypto_op *restrict op; 567 const unsigned int cq_size = qp->entries_n; 568 const unsigned int mask = cq_size - 1; 569 uint32_t idx; 570 uint32_t next_idx = qp->ci & mask; 571 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops); 572 uint16_t i = 0; 573 int ret; 574 575 if (unlikely(max == 0)) 576 return 0; 577 do { 578 idx = next_idx; 579 next_idx = (qp->ci + 1) & mask; 580 op = qp->ops[idx]; 581 cqe = &qp->cq_obj.cqes[idx]; 582 ret = check_cqe(cqe, cq_size, qp->ci); 583 rte_io_rmb(); 584 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 585 if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN)) 586 mlx5_crypto_cqe_err_handle(qp, op); 587 break; 588 } 589 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 590 ops[i++] = op; 591 qp->ci++; 592 } while (i < max); 593 if (likely(i != 0)) { 594 rte_io_wmb(); 595 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci); 596 qp->stats.dequeued_count += i; 597 } 598 return i; 599 } 600 601 static void 602 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp) 603 { 604 uint32_t i; 605 606 for (i = 0 ; i < qp->entries_n; i++) { 607 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i * 608 priv->wqe_set_size); 609 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *) 610 (cseg + 1); 611 struct mlx5_wqe_umr_bsf_seg *bsf = 612 (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg, 613 priv->umr_wqe_size)) - 1; 614 struct mlx5_wqe_rseg *rseg; 615 616 /* Init UMR WQE. */ 617 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | 618 (priv->umr_wqe_size / MLX5_WSEG_SIZE)); 619 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 620 MLX5_COMP_MODE_OFFSET); 621 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id); 622 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET); 623 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */ 624 ucseg->ko_to_bs = rte_cpu_to_be_32 625 ((RTE_ALIGN(priv->max_segs_num, 4u) << 626 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET)); 627 bsf->keytag = priv->keytag; 628 /* Init RDMA WRITE WQE. */ 629 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 630 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS << 631 MLX5_COMP_MODE_OFFSET) | 632 MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE); 633 rseg = (struct mlx5_wqe_rseg *)(cseg + 1); 634 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id); 635 } 636 } 637 638 static int 639 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, 640 struct mlx5_crypto_qp *qp) 641 { 642 struct mlx5_umr_wqe *umr; 643 uint32_t i; 644 struct mlx5_devx_mkey_attr attr = { 645 .pd = priv->pdn, 646 .umr_en = 1, 647 .crypto_en = 1, 648 .set_remote_rw = 1, 649 .klm_num = RTE_ALIGN(priv->max_segs_num, 4), 650 }; 651 652 for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0; 653 i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) { 654 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0]; 655 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr); 656 if (!qp->mkey[i]) 657 goto error; 658 } 659 return 0; 660 error: 661 DRV_LOG(ERR, "Failed to allocate indirect mkey."); 662 mlx5_crypto_indirect_mkeys_release(qp, i); 663 return -1; 664 } 665 666 static int 667 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 668 const struct rte_cryptodev_qp_conf *qp_conf, 669 int socket_id) 670 { 671 struct mlx5_crypto_priv *priv = dev->data->dev_private; 672 struct mlx5_devx_qp_attr attr = {0}; 673 struct mlx5_crypto_qp *qp; 674 uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors); 675 uint32_t umem_size = RTE_BIT32(log_nb_desc) * 676 priv->wqe_set_size + 677 sizeof(*qp->db_rec) * 2; 678 uint32_t alloc_size = sizeof(*qp); 679 struct mlx5_devx_cq_attr cq_attr = { 680 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar), 681 }; 682 683 if (dev->data->queue_pairs[qp_id] != NULL) 684 mlx5_crypto_queue_pair_release(dev, qp_id); 685 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 686 alloc_size += (sizeof(struct rte_crypto_op *) + 687 sizeof(struct mlx5_devx_obj *)) * 688 RTE_BIT32(log_nb_desc); 689 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 690 socket_id); 691 if (qp == NULL) { 692 DRV_LOG(ERR, "Failed to allocate QP memory."); 693 rte_errno = ENOMEM; 694 return -rte_errno; 695 } 696 if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc, 697 &cq_attr, socket_id) != 0) { 698 DRV_LOG(ERR, "Failed to create CQ."); 699 goto error; 700 } 701 qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id); 702 if (qp->umem_buf == NULL) { 703 DRV_LOG(ERR, "Failed to allocate QP umem."); 704 rte_errno = ENOMEM; 705 goto error; 706 } 707 qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx, 708 (void *)(uintptr_t)qp->umem_buf, 709 umem_size, 710 IBV_ACCESS_LOCAL_WRITE); 711 if (qp->umem_obj == NULL) { 712 DRV_LOG(ERR, "Failed to register QP umem."); 713 goto error; 714 } 715 if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N, 716 priv->dev_config.socket_id) != 0) { 717 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 718 (uint32_t)qp_id); 719 rte_errno = ENOMEM; 720 goto error; 721 } 722 qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen; 723 attr.pd = priv->pdn; 724 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar); 725 attr.cqn = qp->cq_obj.cq->id; 726 attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE)); 727 attr.rq_size = 0; 728 attr.sq_size = RTE_BIT32(log_nb_desc); 729 attr.dbr_umem_valid = 1; 730 attr.wq_umem_id = qp->umem_obj->umem_id; 731 attr.wq_umem_offset = 0; 732 attr.dbr_umem_id = qp->umem_obj->umem_id; 733 attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format); 734 attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size; 735 qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr); 736 if (qp->qp_obj == NULL) { 737 DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno); 738 goto error; 739 } 740 qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address); 741 if (mlx5_crypto_qp2rts(qp)) 742 goto error; 743 qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1), 744 RTE_CACHE_LINE_SIZE); 745 qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc)); 746 qp->entries_n = 1 << log_nb_desc; 747 if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) { 748 DRV_LOG(ERR, "Cannot allocate indirect memory regions."); 749 rte_errno = ENOMEM; 750 goto error; 751 } 752 mlx5_crypto_qp_init(priv, qp); 753 qp->priv = priv; 754 dev->data->queue_pairs[qp_id] = qp; 755 return 0; 756 error: 757 mlx5_crypto_qp_release(qp); 758 return -1; 759 } 760 761 static void 762 mlx5_crypto_stats_get(struct rte_cryptodev *dev, 763 struct rte_cryptodev_stats *stats) 764 { 765 int qp_id; 766 767 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 768 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 769 770 stats->enqueued_count += qp->stats.enqueued_count; 771 stats->dequeued_count += qp->stats.dequeued_count; 772 stats->enqueue_err_count += qp->stats.enqueue_err_count; 773 stats->dequeue_err_count += qp->stats.dequeue_err_count; 774 } 775 } 776 777 static void 778 mlx5_crypto_stats_reset(struct rte_cryptodev *dev) 779 { 780 int qp_id; 781 782 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 783 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 784 785 memset(&qp->stats, 0, sizeof(qp->stats)); 786 } 787 } 788 789 static struct rte_cryptodev_ops mlx5_crypto_ops = { 790 .dev_configure = mlx5_crypto_dev_configure, 791 .dev_start = mlx5_crypto_dev_start, 792 .dev_stop = mlx5_crypto_dev_stop, 793 .dev_close = mlx5_crypto_dev_close, 794 .dev_infos_get = mlx5_crypto_dev_infos_get, 795 .stats_get = mlx5_crypto_stats_get, 796 .stats_reset = mlx5_crypto_stats_reset, 797 .queue_pair_setup = mlx5_crypto_queue_pair_setup, 798 .queue_pair_release = mlx5_crypto_queue_pair_release, 799 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 800 .sym_session_configure = mlx5_crypto_sym_session_configure, 801 .sym_session_clear = mlx5_crypto_sym_session_clear, 802 .sym_get_raw_dp_ctx_size = NULL, 803 .sym_configure_raw_dp_ctx = NULL, 804 }; 805 806 static void 807 mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv) 808 { 809 if (priv->pd != NULL) { 810 claim_zero(mlx5_glue->dealloc_pd(priv->pd)); 811 priv->pd = NULL; 812 } 813 if (priv->uar != NULL) { 814 mlx5_glue->devx_free_uar(priv->uar); 815 priv->uar = NULL; 816 } 817 } 818 819 static int 820 mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv) 821 { 822 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 823 struct mlx5dv_obj obj; 824 struct mlx5dv_pd pd_info; 825 int ret; 826 827 priv->pd = mlx5_glue->alloc_pd(priv->ctx); 828 if (priv->pd == NULL) { 829 DRV_LOG(ERR, "Failed to allocate PD."); 830 return errno ? -errno : -ENOMEM; 831 } 832 obj.pd.in = priv->pd; 833 obj.pd.out = &pd_info; 834 ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD); 835 if (ret != 0) { 836 DRV_LOG(ERR, "Fail to get PD object info."); 837 mlx5_glue->dealloc_pd(priv->pd); 838 priv->pd = NULL; 839 return -errno; 840 } 841 priv->pdn = pd_info.pdn; 842 return 0; 843 #else 844 (void)priv; 845 DRV_LOG(ERR, "Cannot get pdn - no DV support."); 846 return -ENOTSUP; 847 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 848 } 849 850 static int 851 mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv) 852 { 853 if (mlx5_crypto_pd_create(priv) != 0) 854 return -1; 855 priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1); 856 if (priv->uar) 857 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar); 858 if (priv->uar == NULL || priv->uar_addr == NULL) { 859 rte_errno = errno; 860 claim_zero(mlx5_glue->dealloc_pd(priv->pd)); 861 DRV_LOG(ERR, "Failed to allocate UAR."); 862 return -1; 863 } 864 return 0; 865 } 866 867 868 static int 869 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 870 { 871 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 872 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 873 unsigned long tmp; 874 FILE *file; 875 int ret; 876 int i; 877 878 if (strcmp(key, "class") == 0) 879 return 0; 880 if (strcmp(key, "wcs_file") == 0) { 881 file = fopen(val, "rb"); 882 if (file == NULL) { 883 rte_errno = ENOTSUP; 884 return -rte_errno; 885 } 886 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 887 ret = fscanf(file, "%02hhX", &attr->credential[i]); 888 if (ret <= 0) { 889 fclose(file); 890 DRV_LOG(ERR, 891 "Failed to read credential from file."); 892 rte_errno = EINVAL; 893 return -rte_errno; 894 } 895 } 896 fclose(file); 897 devarg_prms->login_devarg = true; 898 return 0; 899 } 900 errno = 0; 901 tmp = strtoul(val, NULL, 0); 902 if (errno) { 903 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 904 return -errno; 905 } 906 if (strcmp(key, "max_segs_num") == 0) { 907 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) { 908 DRV_LOG(WARNING, "Invalid max_segs_num: %d, should" 909 " be less than %d.", 910 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS); 911 rte_errno = EINVAL; 912 return -rte_errno; 913 } 914 devarg_prms->max_segs_num = (uint32_t)tmp; 915 } else if (strcmp(key, "import_kek_id") == 0) { 916 attr->session_import_kek_ptr = (uint32_t)tmp; 917 } else if (strcmp(key, "credential_id") == 0) { 918 attr->credential_pointer = (uint32_t)tmp; 919 } else if (strcmp(key, "keytag") == 0) { 920 devarg_prms->keytag = tmp; 921 } else { 922 DRV_LOG(WARNING, "Invalid key %s.", key); 923 } 924 return 0; 925 } 926 927 static int 928 mlx5_crypto_parse_devargs(struct rte_devargs *devargs, 929 struct mlx5_crypto_devarg_params *devarg_prms) 930 { 931 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 932 struct rte_kvargs *kvlist; 933 934 /* Default values. */ 935 attr->credential_pointer = 0; 936 attr->session_import_kek_ptr = 0; 937 devarg_prms->keytag = 0; 938 devarg_prms->max_segs_num = 8; 939 if (devargs == NULL) { 940 DRV_LOG(ERR, 941 "No login devargs in order to enable crypto operations in the device."); 942 rte_errno = EINVAL; 943 return -1; 944 } 945 kvlist = rte_kvargs_parse(devargs->args, NULL); 946 if (kvlist == NULL) { 947 DRV_LOG(ERR, "Failed to parse devargs."); 948 rte_errno = EINVAL; 949 return -1; 950 } 951 if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler, 952 devarg_prms) != 0) { 953 DRV_LOG(ERR, "Devargs handler function Failed."); 954 rte_kvargs_free(kvlist); 955 rte_errno = EINVAL; 956 return -1; 957 } 958 rte_kvargs_free(kvlist); 959 if (devarg_prms->login_devarg == false) { 960 DRV_LOG(ERR, 961 "No login credential devarg in order to enable crypto operations " 962 "in the device."); 963 rte_errno = EINVAL; 964 return -1; 965 } 966 return 0; 967 } 968 969 /** 970 * Callback for memory event. 971 * 972 * @param event_type 973 * Memory event type. 974 * @param addr 975 * Address of memory. 976 * @param len 977 * Size of memory. 978 */ 979 static void 980 mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr, 981 size_t len, void *arg __rte_unused) 982 { 983 struct mlx5_crypto_priv *priv; 984 985 /* Must be called from the primary process. */ 986 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 987 switch (event_type) { 988 case RTE_MEM_EVENT_FREE: 989 pthread_mutex_lock(&priv_list_lock); 990 /* Iterate all the existing mlx5 devices. */ 991 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 992 mlx5_free_mr_by_addr(&priv->mr_scache, 993 priv->ctx->device->name, 994 addr, len); 995 pthread_mutex_unlock(&priv_list_lock); 996 break; 997 case RTE_MEM_EVENT_ALLOC: 998 default: 999 break; 1000 } 1001 } 1002 1003 static int 1004 mlx5_crypto_dev_probe(struct rte_device *dev) 1005 { 1006 struct ibv_device *ibv; 1007 struct rte_cryptodev *crypto_dev; 1008 struct ibv_context *ctx; 1009 struct mlx5_devx_obj *login; 1010 struct mlx5_crypto_priv *priv; 1011 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 1012 struct mlx5_hca_attr attr = { 0 }; 1013 struct rte_cryptodev_pmd_init_params init_params = { 1014 .name = "", 1015 .private_data_size = sizeof(struct mlx5_crypto_priv), 1016 .socket_id = dev->numa_node, 1017 .max_nb_queue_pairs = 1018 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 1019 }; 1020 uint16_t rdmw_wqe_size; 1021 int ret; 1022 1023 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1024 DRV_LOG(ERR, "Non-primary process type is not supported."); 1025 rte_errno = ENOTSUP; 1026 return -rte_errno; 1027 } 1028 ibv = mlx5_os_get_ibv_dev(dev); 1029 if (ibv == NULL) 1030 return -rte_errno; 1031 ctx = mlx5_glue->dv_open_device(ibv); 1032 if (ctx == NULL) { 1033 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name); 1034 rte_errno = ENODEV; 1035 return -rte_errno; 1036 } 1037 if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 || 1038 attr.crypto == 0 || attr.aes_xts == 0) { 1039 DRV_LOG(ERR, "Not enough capabilities to support crypto " 1040 "operations, maybe old FW/OFED version?"); 1041 claim_zero(mlx5_glue->close_device(ctx)); 1042 rte_errno = ENOTSUP; 1043 return -ENOTSUP; 1044 } 1045 ret = mlx5_crypto_parse_devargs(dev->devargs, &devarg_prms); 1046 if (ret) { 1047 DRV_LOG(ERR, "Failed to parse devargs."); 1048 claim_zero(mlx5_glue->close_device(ctx)); 1049 return -rte_errno; 1050 } 1051 login = mlx5_devx_cmd_create_crypto_login_obj(ctx, 1052 &devarg_prms.login_attr); 1053 if (login == NULL) { 1054 DRV_LOG(ERR, "Failed to configure login."); 1055 claim_zero(mlx5_glue->close_device(ctx)); 1056 return -rte_errno; 1057 } 1058 crypto_dev = rte_cryptodev_pmd_create(ibv->name, dev, 1059 &init_params); 1060 if (crypto_dev == NULL) { 1061 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name); 1062 claim_zero(mlx5_glue->close_device(ctx)); 1063 return -ENODEV; 1064 } 1065 DRV_LOG(INFO, 1066 "Crypto device %s was created successfully.", ibv->name); 1067 crypto_dev->dev_ops = &mlx5_crypto_ops; 1068 crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst; 1069 crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst; 1070 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 1071 crypto_dev->driver_id = mlx5_crypto_driver_id; 1072 priv = crypto_dev->data->dev_private; 1073 priv->ctx = ctx; 1074 priv->login_obj = login; 1075 priv->crypto_dev = crypto_dev; 1076 priv->qp_ts_format = attr.qp_ts_format; 1077 if (mlx5_crypto_hw_global_prepare(priv) != 0) { 1078 rte_cryptodev_pmd_destroy(priv->crypto_dev); 1079 claim_zero(mlx5_glue->close_device(priv->ctx)); 1080 return -1; 1081 } 1082 if (mlx5_mr_btree_init(&priv->mr_scache.cache, 1083 MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) { 1084 DRV_LOG(ERR, "Failed to allocate shared cache MR memory."); 1085 mlx5_crypto_hw_global_release(priv); 1086 rte_cryptodev_pmd_destroy(priv->crypto_dev); 1087 claim_zero(mlx5_glue->close_device(priv->ctx)); 1088 rte_errno = ENOMEM; 1089 return -rte_errno; 1090 } 1091 priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr; 1092 priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr; 1093 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 1094 priv->max_segs_num = devarg_prms.max_segs_num; 1095 priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) + 1096 sizeof(struct mlx5_umr_wqe) + 1097 RTE_ALIGN(priv->max_segs_num, 4) * 1098 sizeof(struct mlx5_wqe_dseg); 1099 rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) + 1100 sizeof(struct mlx5_wqe_dseg) * 1101 (priv->max_segs_num <= 2 ? 2 : 2 + 1102 RTE_ALIGN(priv->max_segs_num - 2, 4)); 1103 priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size; 1104 priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB; 1105 priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg); 1106 /* Register callback function for global shared MR cache management. */ 1107 if (TAILQ_EMPTY(&mlx5_crypto_priv_list)) 1108 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", 1109 mlx5_crypto_mr_mem_event_cb, 1110 NULL); 1111 pthread_mutex_lock(&priv_list_lock); 1112 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 1113 pthread_mutex_unlock(&priv_list_lock); 1114 return 0; 1115 } 1116 1117 static int 1118 mlx5_crypto_dev_remove(struct rte_device *dev) 1119 { 1120 struct mlx5_crypto_priv *priv = NULL; 1121 1122 pthread_mutex_lock(&priv_list_lock); 1123 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 1124 if (priv->crypto_dev->device == dev) 1125 break; 1126 if (priv) 1127 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 1128 pthread_mutex_unlock(&priv_list_lock); 1129 if (priv) { 1130 if (TAILQ_EMPTY(&mlx5_crypto_priv_list)) 1131 rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB", 1132 NULL); 1133 mlx5_mr_release_cache(&priv->mr_scache); 1134 mlx5_crypto_hw_global_release(priv); 1135 rte_cryptodev_pmd_destroy(priv->crypto_dev); 1136 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 1137 claim_zero(mlx5_glue->close_device(priv->ctx)); 1138 } 1139 return 0; 1140 } 1141 1142 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 1143 { 1144 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1145 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 1146 }, 1147 { 1148 .vendor_id = 0 1149 } 1150 }; 1151 1152 static struct mlx5_class_driver mlx5_crypto_driver = { 1153 .drv_class = MLX5_CLASS_CRYPTO, 1154 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 1155 .id_table = mlx5_crypto_pci_id_map, 1156 .probe = mlx5_crypto_dev_probe, 1157 .remove = mlx5_crypto_dev_remove, 1158 }; 1159 1160 RTE_INIT(rte_mlx5_crypto_init) 1161 { 1162 mlx5_common_init(); 1163 if (mlx5_glue != NULL) 1164 mlx5_class_driver_register(&mlx5_crypto_driver); 1165 } 1166 1167 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 1168 mlx5_crypto_driver_id); 1169 1170 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 1171 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 1172 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 1173 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 1174