1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_eal_paging.h> 8 #include <rte_errno.h> 9 #include <rte_log.h> 10 #include <rte_bus_pci.h> 11 #include <rte_memory.h> 12 13 #include <mlx5_glue.h> 14 #include <mlx5_common.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_crypto_utils.h" 19 #include "mlx5_crypto.h" 20 21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 23 #define MLX5_CRYPTO_MAX_QPS 1024 24 #define MLX5_CRYPTO_MAX_SEGS 56 25 26 #define MLX5_CRYPTO_FEATURE_FLAGS \ 27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 28 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \ 29 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \ 30 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \ 31 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \ 32 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \ 33 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 34 35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 36 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 37 static pthread_mutex_t priv_list_lock; 38 39 int mlx5_crypto_logtype; 40 41 uint8_t mlx5_crypto_driver_id; 42 43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = { 44 { /* AES XTS */ 45 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, 46 {.sym = { 47 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, 48 {.cipher = { 49 .algo = RTE_CRYPTO_CIPHER_AES_XTS, 50 .block_size = 16, 51 .key_size = { 52 .min = 32, 53 .max = 64, 54 .increment = 32 55 }, 56 .iv_size = { 57 .min = 16, 58 .max = 16, 59 .increment = 0 60 }, 61 .dataunit_set = 62 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES | 63 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES | 64 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES, 65 }, } 66 }, } 67 }, 68 }; 69 70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 71 72 static const struct rte_driver mlx5_drv = { 73 .name = mlx5_crypto_drv_name, 74 .alias = mlx5_crypto_drv_name 75 }; 76 77 static struct cryptodev_driver mlx5_cryptodev_driver; 78 79 struct mlx5_crypto_session { 80 uint32_t bs_bpt_eo_es; 81 /**< bsf_size, bsf_p_type, encryption_order and encryption standard, 82 * saved in big endian format. 83 */ 84 uint32_t bsp_res; 85 /**< crypto_block_size_pointer and reserved 24 bits saved in big 86 * endian format. 87 */ 88 uint32_t iv_offset:16; 89 /**< Starting point for Initialisation Vector. */ 90 struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */ 91 uint32_t dek_id; /**< DEK ID */ 92 } __rte_packed; 93 94 static void 95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 96 struct rte_cryptodev_info *dev_info) 97 { 98 RTE_SET_USED(dev); 99 if (dev_info != NULL) { 100 dev_info->driver_id = mlx5_crypto_driver_id; 101 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 102 dev_info->capabilities = mlx5_crypto_caps; 103 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 104 dev_info->min_mbuf_headroom_req = 0; 105 dev_info->min_mbuf_tailroom_req = 0; 106 dev_info->sym.max_nb_sessions = 0; 107 /* 108 * If 0, the device does not have any limitation in number of 109 * sessions that can be used. 110 */ 111 } 112 } 113 114 static int 115 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 116 struct rte_cryptodev_config *config) 117 { 118 struct mlx5_crypto_priv *priv = dev->data->dev_private; 119 120 if (config == NULL) { 121 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 122 return -EINVAL; 123 } 124 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 125 DRV_LOG(ERR, 126 "Disabled symmetric crypto feature is not supported."); 127 return -ENOTSUP; 128 } 129 if (mlx5_crypto_dek_setup(priv) != 0) { 130 DRV_LOG(ERR, "Dek hash list creation has failed."); 131 return -ENOMEM; 132 } 133 priv->dev_config = *config; 134 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 135 return 0; 136 } 137 138 static void 139 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 140 { 141 RTE_SET_USED(dev); 142 } 143 144 static int 145 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 146 { 147 struct mlx5_crypto_priv *priv = dev->data->dev_private; 148 149 return mlx5_dev_mempool_subscribe(priv->cdev); 150 } 151 152 static int 153 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 154 { 155 struct mlx5_crypto_priv *priv = dev->data->dev_private; 156 157 mlx5_crypto_dek_unset(priv); 158 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 159 return 0; 160 } 161 162 static unsigned int 163 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 164 { 165 return sizeof(struct mlx5_crypto_session); 166 } 167 168 static int 169 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev, 170 struct rte_crypto_sym_xform *xform, 171 struct rte_cryptodev_sym_session *session, 172 struct rte_mempool *mp) 173 { 174 struct mlx5_crypto_priv *priv = dev->data->dev_private; 175 struct mlx5_crypto_session *sess_private_data; 176 struct rte_crypto_cipher_xform *cipher; 177 uint8_t encryption_order; 178 int ret; 179 180 if (unlikely(xform->next != NULL)) { 181 DRV_LOG(ERR, "Xform next is not supported."); 182 return -ENOTSUP; 183 } 184 if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) || 185 (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) { 186 DRV_LOG(ERR, "Only AES-XTS algorithm is supported."); 187 return -ENOTSUP; 188 } 189 ret = rte_mempool_get(mp, (void *)&sess_private_data); 190 if (ret != 0) { 191 DRV_LOG(ERR, 192 "Failed to get session %p private data from mempool.", 193 sess_private_data); 194 return -ENOMEM; 195 } 196 cipher = &xform->cipher; 197 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher); 198 if (sess_private_data->dek == NULL) { 199 rte_mempool_put(mp, sess_private_data); 200 DRV_LOG(ERR, "Failed to prepare dek."); 201 return -ENOMEM; 202 } 203 if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 204 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY; 205 else 206 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE; 207 sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32 208 (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET | 209 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET | 210 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET | 211 MLX5_ENCRYPTION_STANDARD_AES_XTS); 212 switch (xform->cipher.dataunit_len) { 213 case 0: 214 sess_private_data->bsp_res = 0; 215 break; 216 case 512: 217 sess_private_data->bsp_res = rte_cpu_to_be_32 218 ((uint32_t)MLX5_BLOCK_SIZE_512B << 219 MLX5_BLOCK_SIZE_OFFSET); 220 break; 221 case 4096: 222 sess_private_data->bsp_res = rte_cpu_to_be_32 223 ((uint32_t)MLX5_BLOCK_SIZE_4096B << 224 MLX5_BLOCK_SIZE_OFFSET); 225 break; 226 case 1048576: 227 sess_private_data->bsp_res = rte_cpu_to_be_32 228 ((uint32_t)MLX5_BLOCK_SIZE_1MB << 229 MLX5_BLOCK_SIZE_OFFSET); 230 break; 231 default: 232 DRV_LOG(ERR, "Cipher data unit length is not supported."); 233 return -ENOTSUP; 234 } 235 sess_private_data->iv_offset = cipher->iv.offset; 236 sess_private_data->dek_id = 237 rte_cpu_to_be_32(sess_private_data->dek->obj->id & 238 0xffffff); 239 set_sym_session_private_data(session, dev->driver_id, 240 sess_private_data); 241 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data); 242 return 0; 243 } 244 245 static void 246 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 247 struct rte_cryptodev_sym_session *sess) 248 { 249 struct mlx5_crypto_priv *priv = dev->data->dev_private; 250 struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess, 251 dev->driver_id); 252 253 if (unlikely(spriv == NULL)) { 254 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 255 return; 256 } 257 mlx5_crypto_dek_destroy(priv, spriv->dek); 258 set_sym_session_private_data(sess, dev->driver_id, NULL); 259 rte_mempool_put(rte_mempool_from_obj(spriv), spriv); 260 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 261 } 262 263 static void 264 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n) 265 { 266 uint16_t i; 267 268 for (i = 0; i < n; i++) 269 if (qp->mkey[i]) 270 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i])); 271 } 272 273 static void 274 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp) 275 { 276 if (qp == NULL) 277 return; 278 mlx5_devx_qp_destroy(&qp->qp_obj); 279 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 280 mlx5_devx_cq_destroy(&qp->cq_obj); 281 rte_free(qp); 282 } 283 284 static int 285 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id) 286 { 287 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 288 289 mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n); 290 mlx5_crypto_qp_release(qp); 291 dev->data->queue_pairs[qp_id] = NULL; 292 return 0; 293 } 294 295 static __rte_noinline uint32_t 296 mlx5_crypto_get_block_size(struct rte_crypto_op *op) 297 { 298 uint32_t bl = op->sym->cipher.data.length; 299 300 switch (bl) { 301 case (1 << 20): 302 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET); 303 case (1 << 12): 304 return RTE_BE32(MLX5_BLOCK_SIZE_4096B << 305 MLX5_BLOCK_SIZE_OFFSET); 306 case (1 << 9): 307 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET); 308 default: 309 DRV_LOG(ERR, "Unknown block size: %u.", bl); 310 return UINT32_MAX; 311 } 312 } 313 314 static __rte_always_inline uint32_t 315 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp, 316 struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm, 317 uint32_t offset, uint32_t *remain) 318 { 319 uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset); 320 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset); 321 322 if (data_len > *remain) 323 data_len = *remain; 324 *remain -= data_len; 325 klm->bcount = rte_cpu_to_be_32(data_len); 326 klm->pbuf = rte_cpu_to_be_64(addr); 327 klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf); 328 return klm->lkey; 329 330 } 331 332 static __rte_always_inline uint32_t 333 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp, 334 struct rte_crypto_op *op, struct rte_mbuf *mbuf, 335 struct mlx5_wqe_dseg *klm) 336 { 337 uint32_t remain_len = op->sym->cipher.data.length; 338 uint32_t nb_segs = mbuf->nb_segs; 339 uint32_t klm_n = 1u; 340 341 /* First mbuf needs to take the cipher offset. */ 342 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm, 343 op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) { 344 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 345 return 0; 346 } 347 while (remain_len) { 348 nb_segs--; 349 mbuf = mbuf->next; 350 if (unlikely(mbuf == NULL || nb_segs == 0)) { 351 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 352 return 0; 353 } 354 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0, 355 &remain_len) == UINT32_MAX)) { 356 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 357 return 0; 358 } 359 klm_n++; 360 } 361 return klm_n; 362 } 363 364 static __rte_always_inline int 365 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv, 366 struct mlx5_crypto_qp *qp, 367 struct rte_crypto_op *op, 368 struct mlx5_umr_wqe *umr) 369 { 370 struct mlx5_crypto_session *sess = get_sym_session_private_data 371 (op->sym->session, mlx5_crypto_driver_id); 372 struct mlx5_wqe_cseg *cseg = &umr->ctr; 373 struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc; 374 struct mlx5_wqe_dseg *klms = &umr->kseg[0]; 375 struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *) 376 RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1; 377 uint32_t ds; 378 bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src; 379 /* Set UMR WQE. */ 380 uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op, 381 ipl ? op->sym->m_src : op->sym->m_dst, klms); 382 383 if (unlikely(klm_n == 0)) 384 return 0; 385 bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es; 386 if (unlikely(!sess->bsp_res)) { 387 bsf->bsp_res = mlx5_crypto_get_block_size(op); 388 if (unlikely(bsf->bsp_res == UINT32_MAX)) { 389 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 390 return 0; 391 } 392 } else { 393 bsf->bsp_res = sess->bsp_res; 394 } 395 bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length); 396 memcpy(bsf->xts_initial_tweak, 397 rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16); 398 bsf->res_dp = sess->dek_id; 399 mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length); 400 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR); 401 qp->db_pi += priv->umr_wqe_stride; 402 /* Set RDMA_WRITE WQE. */ 403 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 404 klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe)); 405 if (!ipl) { 406 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src, 407 klms); 408 if (unlikely(klm_n == 0)) 409 return 0; 410 } else { 411 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n); 412 } 413 ds = 2 + klm_n; 414 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds); 415 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 416 MLX5_OPCODE_RDMA_WRITE); 417 ds = RTE_ALIGN(ds, 4); 418 qp->db_pi += ds >> 2; 419 /* Set NOP WQE if needed. */ 420 if (priv->max_rdmar_ds > ds) { 421 cseg += ds; 422 ds = priv->max_rdmar_ds - ds; 423 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds); 424 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 425 MLX5_OPCODE_NOP); 426 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */ 427 } 428 qp->wqe = (uint8_t *)cseg; 429 return 1; 430 } 431 432 static __rte_always_inline void 433 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv) 434 { 435 #ifdef RTE_ARCH_64 436 *priv->uar_addr = val; 437 #else /* !RTE_ARCH_64 */ 438 rte_spinlock_lock(&priv->uar32_sl); 439 *(volatile uint32_t *)priv->uar_addr = val; 440 rte_io_wmb(); 441 *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32; 442 rte_spinlock_unlock(&priv->uar32_sl); 443 #endif 444 } 445 446 static uint16_t 447 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 448 uint16_t nb_ops) 449 { 450 struct mlx5_crypto_qp *qp = queue_pair; 451 struct mlx5_crypto_priv *priv = qp->priv; 452 struct mlx5_umr_wqe *umr; 453 struct rte_crypto_op *op; 454 uint16_t mask = qp->entries_n - 1; 455 uint16_t remain = qp->entries_n - (qp->pi - qp->ci); 456 uint32_t idx; 457 458 if (remain < nb_ops) 459 nb_ops = remain; 460 else 461 remain = nb_ops; 462 if (unlikely(remain == 0)) 463 return 0; 464 do { 465 idx = qp->pi & mask; 466 op = *ops++; 467 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf, 468 priv->wqe_set_size * idx); 469 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) { 470 qp->stats.enqueue_err_count++; 471 if (remain != nb_ops) { 472 qp->stats.enqueued_count -= remain; 473 break; 474 } 475 return 0; 476 } 477 qp->ops[idx] = op; 478 qp->pi++; 479 } while (--remain); 480 qp->stats.enqueued_count += nb_ops; 481 rte_io_wmb(); 482 qp->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi); 483 rte_wmb(); 484 mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv); 485 rte_wmb(); 486 return nb_ops; 487 } 488 489 static __rte_noinline void 490 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op) 491 { 492 const uint32_t idx = qp->ci & (qp->entries_n - 1); 493 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *) 494 &qp->cq_obj.cqes[idx]; 495 496 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 497 qp->stats.dequeue_err_count++; 498 DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome)); 499 } 500 501 static uint16_t 502 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 503 uint16_t nb_ops) 504 { 505 struct mlx5_crypto_qp *qp = queue_pair; 506 volatile struct mlx5_cqe *restrict cqe; 507 struct rte_crypto_op *restrict op; 508 const unsigned int cq_size = qp->entries_n; 509 const unsigned int mask = cq_size - 1; 510 uint32_t idx; 511 uint32_t next_idx = qp->ci & mask; 512 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops); 513 uint16_t i = 0; 514 int ret; 515 516 if (unlikely(max == 0)) 517 return 0; 518 do { 519 idx = next_idx; 520 next_idx = (qp->ci + 1) & mask; 521 op = qp->ops[idx]; 522 cqe = &qp->cq_obj.cqes[idx]; 523 ret = check_cqe(cqe, cq_size, qp->ci); 524 rte_io_rmb(); 525 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 526 if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN)) 527 mlx5_crypto_cqe_err_handle(qp, op); 528 break; 529 } 530 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 531 ops[i++] = op; 532 qp->ci++; 533 } while (i < max); 534 if (likely(i != 0)) { 535 rte_io_wmb(); 536 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci); 537 qp->stats.dequeued_count += i; 538 } 539 return i; 540 } 541 542 static void 543 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp) 544 { 545 uint32_t i; 546 547 for (i = 0 ; i < qp->entries_n; i++) { 548 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf, 549 i * priv->wqe_set_size); 550 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *) 551 (cseg + 1); 552 struct mlx5_wqe_umr_bsf_seg *bsf = 553 (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg, 554 priv->umr_wqe_size)) - 1; 555 struct mlx5_wqe_rseg *rseg; 556 557 /* Init UMR WQE. */ 558 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | 559 (priv->umr_wqe_size / MLX5_WSEG_SIZE)); 560 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 561 MLX5_COMP_MODE_OFFSET); 562 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id); 563 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET); 564 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */ 565 ucseg->ko_to_bs = rte_cpu_to_be_32 566 ((RTE_ALIGN(priv->max_segs_num, 4u) << 567 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET)); 568 bsf->keytag = priv->keytag; 569 /* Init RDMA WRITE WQE. */ 570 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 571 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS << 572 MLX5_COMP_MODE_OFFSET) | 573 MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE); 574 rseg = (struct mlx5_wqe_rseg *)(cseg + 1); 575 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id); 576 } 577 } 578 579 static int 580 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, 581 struct mlx5_crypto_qp *qp) 582 { 583 struct mlx5_umr_wqe *umr; 584 uint32_t i; 585 struct mlx5_devx_mkey_attr attr = { 586 .pd = priv->cdev->pdn, 587 .umr_en = 1, 588 .crypto_en = 1, 589 .set_remote_rw = 1, 590 .klm_num = RTE_ALIGN(priv->max_segs_num, 4), 591 }; 592 593 for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0; 594 i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) { 595 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0]; 596 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr); 597 if (!qp->mkey[i]) 598 goto error; 599 } 600 return 0; 601 error: 602 DRV_LOG(ERR, "Failed to allocate indirect mkey."); 603 mlx5_crypto_indirect_mkeys_release(qp, i); 604 return -1; 605 } 606 607 static int 608 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 609 const struct rte_cryptodev_qp_conf *qp_conf, 610 int socket_id) 611 { 612 struct mlx5_crypto_priv *priv = dev->data->dev_private; 613 struct mlx5_devx_qp_attr attr = {0}; 614 struct mlx5_crypto_qp *qp; 615 uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors); 616 uint32_t ret; 617 uint32_t alloc_size = sizeof(*qp); 618 struct mlx5_devx_cq_attr cq_attr = { 619 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar), 620 }; 621 622 if (dev->data->queue_pairs[qp_id] != NULL) 623 mlx5_crypto_queue_pair_release(dev, qp_id); 624 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 625 alloc_size += (sizeof(struct rte_crypto_op *) + 626 sizeof(struct mlx5_devx_obj *)) * 627 RTE_BIT32(log_nb_desc); 628 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 629 socket_id); 630 if (qp == NULL) { 631 DRV_LOG(ERR, "Failed to allocate QP memory."); 632 rte_errno = ENOMEM; 633 return -rte_errno; 634 } 635 if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc, 636 &cq_attr, socket_id) != 0) { 637 DRV_LOG(ERR, "Failed to create CQ."); 638 goto error; 639 } 640 attr.pd = priv->cdev->pdn; 641 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar); 642 attr.cqn = qp->cq_obj.cq->id; 643 attr.rq_size = 0; 644 attr.sq_size = RTE_BIT32(log_nb_desc); 645 attr.ts_format = 646 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format); 647 ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc, 648 &attr, socket_id); 649 if (ret) { 650 DRV_LOG(ERR, "Failed to create QP."); 651 goto error; 652 } 653 if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen, 654 priv->dev_config.socket_id) != 0) { 655 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 656 (uint32_t)qp_id); 657 rte_errno = ENOMEM; 658 goto error; 659 } 660 /* 661 * In Order to configure self loopback, when calling devx qp2rts the 662 * remote QP id that is used is the id of the same QP. 663 */ 664 if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id)) 665 goto error; 666 qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1), 667 RTE_CACHE_LINE_SIZE); 668 qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc)); 669 qp->entries_n = 1 << log_nb_desc; 670 if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) { 671 DRV_LOG(ERR, "Cannot allocate indirect memory regions."); 672 rte_errno = ENOMEM; 673 goto error; 674 } 675 mlx5_crypto_qp_init(priv, qp); 676 qp->priv = priv; 677 dev->data->queue_pairs[qp_id] = qp; 678 return 0; 679 error: 680 mlx5_crypto_qp_release(qp); 681 return -1; 682 } 683 684 static void 685 mlx5_crypto_stats_get(struct rte_cryptodev *dev, 686 struct rte_cryptodev_stats *stats) 687 { 688 int qp_id; 689 690 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 691 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 692 693 stats->enqueued_count += qp->stats.enqueued_count; 694 stats->dequeued_count += qp->stats.dequeued_count; 695 stats->enqueue_err_count += qp->stats.enqueue_err_count; 696 stats->dequeue_err_count += qp->stats.dequeue_err_count; 697 } 698 } 699 700 static void 701 mlx5_crypto_stats_reset(struct rte_cryptodev *dev) 702 { 703 int qp_id; 704 705 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 706 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 707 708 memset(&qp->stats, 0, sizeof(qp->stats)); 709 } 710 } 711 712 static struct rte_cryptodev_ops mlx5_crypto_ops = { 713 .dev_configure = mlx5_crypto_dev_configure, 714 .dev_start = mlx5_crypto_dev_start, 715 .dev_stop = mlx5_crypto_dev_stop, 716 .dev_close = mlx5_crypto_dev_close, 717 .dev_infos_get = mlx5_crypto_dev_infos_get, 718 .stats_get = mlx5_crypto_stats_get, 719 .stats_reset = mlx5_crypto_stats_reset, 720 .queue_pair_setup = mlx5_crypto_queue_pair_setup, 721 .queue_pair_release = mlx5_crypto_queue_pair_release, 722 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 723 .sym_session_configure = mlx5_crypto_sym_session_configure, 724 .sym_session_clear = mlx5_crypto_sym_session_clear, 725 .sym_get_raw_dp_ctx_size = NULL, 726 .sym_configure_raw_dp_ctx = NULL, 727 }; 728 729 static void 730 mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv) 731 { 732 if (priv->uar != NULL) { 733 mlx5_glue->devx_free_uar(priv->uar); 734 priv->uar = NULL; 735 } 736 } 737 738 static int 739 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv) 740 { 741 priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1); 742 if (priv->uar) 743 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar); 744 if (priv->uar == NULL || priv->uar_addr == NULL) { 745 rte_errno = errno; 746 DRV_LOG(ERR, "Failed to allocate UAR."); 747 return -1; 748 } 749 return 0; 750 } 751 752 753 static int 754 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 755 { 756 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 757 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 758 unsigned long tmp; 759 FILE *file; 760 int ret; 761 int i; 762 763 if (strcmp(key, "class") == 0) 764 return 0; 765 if (strcmp(key, "wcs_file") == 0) { 766 file = fopen(val, "rb"); 767 if (file == NULL) { 768 rte_errno = ENOTSUP; 769 return -rte_errno; 770 } 771 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 772 ret = fscanf(file, "%02hhX", &attr->credential[i]); 773 if (ret <= 0) { 774 fclose(file); 775 DRV_LOG(ERR, 776 "Failed to read credential from file."); 777 rte_errno = EINVAL; 778 return -rte_errno; 779 } 780 } 781 fclose(file); 782 devarg_prms->login_devarg = true; 783 return 0; 784 } 785 errno = 0; 786 tmp = strtoul(val, NULL, 0); 787 if (errno) { 788 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 789 return -errno; 790 } 791 if (strcmp(key, "max_segs_num") == 0) { 792 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) { 793 DRV_LOG(WARNING, "Invalid max_segs_num: %d, should" 794 " be less than %d.", 795 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS); 796 rte_errno = EINVAL; 797 return -rte_errno; 798 } 799 devarg_prms->max_segs_num = (uint32_t)tmp; 800 } else if (strcmp(key, "import_kek_id") == 0) { 801 attr->session_import_kek_ptr = (uint32_t)tmp; 802 } else if (strcmp(key, "credential_id") == 0) { 803 attr->credential_pointer = (uint32_t)tmp; 804 } else if (strcmp(key, "keytag") == 0) { 805 devarg_prms->keytag = tmp; 806 } else { 807 DRV_LOG(WARNING, "Invalid key %s.", key); 808 } 809 return 0; 810 } 811 812 static int 813 mlx5_crypto_parse_devargs(struct rte_devargs *devargs, 814 struct mlx5_crypto_devarg_params *devarg_prms) 815 { 816 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 817 struct rte_kvargs *kvlist; 818 819 /* Default values. */ 820 attr->credential_pointer = 0; 821 attr->session_import_kek_ptr = 0; 822 devarg_prms->keytag = 0; 823 devarg_prms->max_segs_num = 8; 824 if (devargs == NULL) { 825 DRV_LOG(ERR, 826 "No login devargs in order to enable crypto operations in the device."); 827 rte_errno = EINVAL; 828 return -1; 829 } 830 kvlist = rte_kvargs_parse(devargs->args, NULL); 831 if (kvlist == NULL) { 832 DRV_LOG(ERR, "Failed to parse devargs."); 833 rte_errno = EINVAL; 834 return -1; 835 } 836 if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler, 837 devarg_prms) != 0) { 838 DRV_LOG(ERR, "Devargs handler function Failed."); 839 rte_kvargs_free(kvlist); 840 rte_errno = EINVAL; 841 return -1; 842 } 843 rte_kvargs_free(kvlist); 844 if (devarg_prms->login_devarg == false) { 845 DRV_LOG(ERR, 846 "No login credential devarg in order to enable crypto operations " 847 "in the device."); 848 rte_errno = EINVAL; 849 return -1; 850 } 851 return 0; 852 } 853 854 static int 855 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev) 856 { 857 struct rte_cryptodev *crypto_dev; 858 struct mlx5_devx_obj *login; 859 struct mlx5_crypto_priv *priv; 860 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 861 struct rte_cryptodev_pmd_init_params init_params = { 862 .name = "", 863 .private_data_size = sizeof(struct mlx5_crypto_priv), 864 .socket_id = cdev->dev->numa_node, 865 .max_nb_queue_pairs = 866 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 867 }; 868 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx); 869 uint16_t rdmw_wqe_size; 870 int ret; 871 872 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 873 DRV_LOG(ERR, "Non-primary process type is not supported."); 874 rte_errno = ENOTSUP; 875 return -rte_errno; 876 } 877 if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) { 878 DRV_LOG(ERR, "Not enough capabilities to support crypto " 879 "operations, maybe old FW/OFED version?"); 880 rte_errno = ENOTSUP; 881 return -ENOTSUP; 882 } 883 ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms); 884 if (ret) { 885 DRV_LOG(ERR, "Failed to parse devargs."); 886 return -rte_errno; 887 } 888 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx, 889 &devarg_prms.login_attr); 890 if (login == NULL) { 891 DRV_LOG(ERR, "Failed to configure login."); 892 return -rte_errno; 893 } 894 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev, 895 &init_params); 896 if (crypto_dev == NULL) { 897 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name); 898 return -ENODEV; 899 } 900 DRV_LOG(INFO, 901 "Crypto device %s was created successfully.", ibdev_name); 902 crypto_dev->dev_ops = &mlx5_crypto_ops; 903 crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst; 904 crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst; 905 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 906 crypto_dev->driver_id = mlx5_crypto_driver_id; 907 priv = crypto_dev->data->dev_private; 908 priv->cdev = cdev; 909 priv->login_obj = login; 910 priv->crypto_dev = crypto_dev; 911 if (mlx5_crypto_uar_prepare(priv) != 0) { 912 rte_cryptodev_pmd_destroy(priv->crypto_dev); 913 return -1; 914 } 915 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 916 priv->max_segs_num = devarg_prms.max_segs_num; 917 priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) + 918 sizeof(struct mlx5_wqe_cseg) + 919 sizeof(struct mlx5_wqe_umr_cseg) + 920 sizeof(struct mlx5_wqe_mkey_cseg) + 921 RTE_ALIGN(priv->max_segs_num, 4) * 922 sizeof(struct mlx5_wqe_dseg); 923 rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) + 924 sizeof(struct mlx5_wqe_dseg) * 925 (priv->max_segs_num <= 2 ? 2 : 2 + 926 RTE_ALIGN(priv->max_segs_num - 2, 4)); 927 priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size; 928 priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB; 929 priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg); 930 pthread_mutex_lock(&priv_list_lock); 931 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 932 pthread_mutex_unlock(&priv_list_lock); 933 934 rte_cryptodev_pmd_probing_finish(crypto_dev); 935 936 return 0; 937 } 938 939 static int 940 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev) 941 { 942 struct mlx5_crypto_priv *priv = NULL; 943 944 pthread_mutex_lock(&priv_list_lock); 945 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 946 if (priv->crypto_dev->device == cdev->dev) 947 break; 948 if (priv) 949 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 950 pthread_mutex_unlock(&priv_list_lock); 951 if (priv) { 952 mlx5_crypto_uar_release(priv); 953 rte_cryptodev_pmd_destroy(priv->crypto_dev); 954 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 955 } 956 return 0; 957 } 958 959 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 960 { 961 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 962 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 963 }, 964 { 965 .vendor_id = 0 966 } 967 }; 968 969 static struct mlx5_class_driver mlx5_crypto_driver = { 970 .drv_class = MLX5_CLASS_CRYPTO, 971 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 972 .id_table = mlx5_crypto_pci_id_map, 973 .probe = mlx5_crypto_dev_probe, 974 .remove = mlx5_crypto_dev_remove, 975 }; 976 977 RTE_INIT(rte_mlx5_crypto_init) 978 { 979 pthread_mutex_init(&priv_list_lock, NULL); 980 mlx5_common_init(); 981 if (mlx5_glue != NULL) 982 mlx5_class_driver_register(&mlx5_crypto_driver); 983 } 984 985 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 986 mlx5_crypto_driver_id); 987 988 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 989 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 990 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 991 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 992