1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_errno.h> 8 #include <rte_log.h> 9 #include <rte_bus_pci.h> 10 #include <rte_memory.h> 11 12 #include <mlx5_glue.h> 13 #include <mlx5_common.h> 14 #include <mlx5_devx_cmds.h> 15 #include <mlx5_common_os.h> 16 17 #include "mlx5_crypto_utils.h" 18 #include "mlx5_crypto.h" 19 20 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 21 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 22 #define MLX5_CRYPTO_MAX_QPS 1024 23 #define MLX5_CRYPTO_MAX_SEGS 56 24 25 #define MLX5_CRYPTO_FEATURE_FLAGS \ 26 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 27 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \ 28 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \ 29 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \ 30 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \ 31 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \ 32 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 33 34 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 35 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 36 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER; 37 38 int mlx5_crypto_logtype; 39 40 uint8_t mlx5_crypto_driver_id; 41 42 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = { 43 { /* AES XTS */ 44 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, 45 {.sym = { 46 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, 47 {.cipher = { 48 .algo = RTE_CRYPTO_CIPHER_AES_XTS, 49 .block_size = 16, 50 .key_size = { 51 .min = 32, 52 .max = 64, 53 .increment = 32 54 }, 55 .iv_size = { 56 .min = 16, 57 .max = 16, 58 .increment = 0 59 }, 60 .dataunit_set = 61 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES | 62 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES, 63 }, } 64 }, } 65 }, 66 }; 67 68 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 69 70 static const struct rte_driver mlx5_drv = { 71 .name = mlx5_crypto_drv_name, 72 .alias = mlx5_crypto_drv_name 73 }; 74 75 static struct cryptodev_driver mlx5_cryptodev_driver; 76 77 struct mlx5_crypto_session { 78 uint32_t bs_bpt_eo_es; 79 /**< bsf_size, bsf_p_type, encryption_order and encryption standard, 80 * saved in big endian format. 81 */ 82 uint32_t bsp_res; 83 /**< crypto_block_size_pointer and reserved 24 bits saved in big 84 * endian format. 85 */ 86 uint32_t iv_offset:16; 87 /**< Starting point for Initialisation Vector. */ 88 struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */ 89 uint32_t dek_id; /**< DEK ID */ 90 } __rte_packed; 91 92 static void 93 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 94 struct rte_cryptodev_info *dev_info) 95 { 96 RTE_SET_USED(dev); 97 if (dev_info != NULL) { 98 dev_info->driver_id = mlx5_crypto_driver_id; 99 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 100 dev_info->capabilities = mlx5_crypto_caps; 101 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 102 dev_info->min_mbuf_headroom_req = 0; 103 dev_info->min_mbuf_tailroom_req = 0; 104 dev_info->sym.max_nb_sessions = 0; 105 /* 106 * If 0, the device does not have any limitation in number of 107 * sessions that can be used. 108 */ 109 } 110 } 111 112 static int 113 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 114 struct rte_cryptodev_config *config) 115 { 116 struct mlx5_crypto_priv *priv = dev->data->dev_private; 117 118 if (config == NULL) { 119 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 120 return -EINVAL; 121 } 122 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 123 DRV_LOG(ERR, 124 "Disabled symmetric crypto feature is not supported."); 125 return -ENOTSUP; 126 } 127 if (mlx5_crypto_dek_setup(priv) != 0) { 128 DRV_LOG(ERR, "Dek hash list creation has failed."); 129 return -ENOMEM; 130 } 131 priv->dev_config = *config; 132 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 133 return 0; 134 } 135 136 static void 137 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 138 { 139 RTE_SET_USED(dev); 140 } 141 142 static int 143 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 144 { 145 struct mlx5_crypto_priv *priv = dev->data->dev_private; 146 147 return mlx5_dev_mempool_subscribe(priv->cdev); 148 } 149 150 static int 151 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 152 { 153 struct mlx5_crypto_priv *priv = dev->data->dev_private; 154 155 mlx5_crypto_dek_unset(priv); 156 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 157 return 0; 158 } 159 160 static unsigned int 161 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 162 { 163 return sizeof(struct mlx5_crypto_session); 164 } 165 166 static int 167 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev, 168 struct rte_crypto_sym_xform *xform, 169 struct rte_cryptodev_sym_session *session, 170 struct rte_mempool *mp) 171 { 172 struct mlx5_crypto_priv *priv = dev->data->dev_private; 173 struct mlx5_crypto_session *sess_private_data; 174 struct rte_crypto_cipher_xform *cipher; 175 uint8_t encryption_order; 176 int ret; 177 178 if (unlikely(xform->next != NULL)) { 179 DRV_LOG(ERR, "Xform next is not supported."); 180 return -ENOTSUP; 181 } 182 if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) || 183 (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) { 184 DRV_LOG(ERR, "Only AES-XTS algorithm is supported."); 185 return -ENOTSUP; 186 } 187 ret = rte_mempool_get(mp, (void *)&sess_private_data); 188 if (ret != 0) { 189 DRV_LOG(ERR, 190 "Failed to get session %p private data from mempool.", 191 sess_private_data); 192 return -ENOMEM; 193 } 194 cipher = &xform->cipher; 195 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher); 196 if (sess_private_data->dek == NULL) { 197 rte_mempool_put(mp, sess_private_data); 198 DRV_LOG(ERR, "Failed to prepare dek."); 199 return -ENOMEM; 200 } 201 if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 202 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY; 203 else 204 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE; 205 sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32 206 (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET | 207 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET | 208 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET | 209 MLX5_ENCRYPTION_STANDARD_AES_XTS); 210 switch (xform->cipher.dataunit_len) { 211 case 0: 212 sess_private_data->bsp_res = 0; 213 break; 214 case 512: 215 sess_private_data->bsp_res = rte_cpu_to_be_32 216 ((uint32_t)MLX5_BLOCK_SIZE_512B << 217 MLX5_BLOCK_SIZE_OFFSET); 218 break; 219 case 4096: 220 sess_private_data->bsp_res = rte_cpu_to_be_32 221 ((uint32_t)MLX5_BLOCK_SIZE_4096B << 222 MLX5_BLOCK_SIZE_OFFSET); 223 break; 224 default: 225 DRV_LOG(ERR, "Cipher data unit length is not supported."); 226 return -ENOTSUP; 227 } 228 sess_private_data->iv_offset = cipher->iv.offset; 229 sess_private_data->dek_id = 230 rte_cpu_to_be_32(sess_private_data->dek->obj->id & 231 0xffffff); 232 set_sym_session_private_data(session, dev->driver_id, 233 sess_private_data); 234 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data); 235 return 0; 236 } 237 238 static void 239 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 240 struct rte_cryptodev_sym_session *sess) 241 { 242 struct mlx5_crypto_priv *priv = dev->data->dev_private; 243 struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess, 244 dev->driver_id); 245 246 if (unlikely(spriv == NULL)) { 247 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 248 return; 249 } 250 mlx5_crypto_dek_destroy(priv, spriv->dek); 251 set_sym_session_private_data(sess, dev->driver_id, NULL); 252 rte_mempool_put(rte_mempool_from_obj(spriv), spriv); 253 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 254 } 255 256 static void 257 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n) 258 { 259 uint16_t i; 260 261 for (i = 0; i < n; i++) 262 if (qp->mkey[i]) 263 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i])); 264 } 265 266 static void 267 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp) 268 { 269 if (qp == NULL) 270 return; 271 mlx5_devx_qp_destroy(&qp->qp_obj); 272 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 273 mlx5_devx_cq_destroy(&qp->cq_obj); 274 rte_free(qp); 275 } 276 277 static int 278 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id) 279 { 280 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 281 282 mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n); 283 mlx5_crypto_qp_release(qp); 284 dev->data->queue_pairs[qp_id] = NULL; 285 return 0; 286 } 287 288 static __rte_noinline uint32_t 289 mlx5_crypto_get_block_size(struct rte_crypto_op *op) 290 { 291 uint32_t bl = op->sym->cipher.data.length; 292 293 switch (bl) { 294 case (1 << 20): 295 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET); 296 case (1 << 12): 297 return RTE_BE32(MLX5_BLOCK_SIZE_4096B << 298 MLX5_BLOCK_SIZE_OFFSET); 299 case (1 << 9): 300 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET); 301 default: 302 DRV_LOG(ERR, "Unknown block size: %u.", bl); 303 return UINT32_MAX; 304 } 305 } 306 307 static __rte_always_inline uint32_t 308 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp, 309 struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm, 310 uint32_t offset, uint32_t *remain) 311 { 312 uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset); 313 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset); 314 315 if (data_len > *remain) 316 data_len = *remain; 317 *remain -= data_len; 318 klm->bcount = rte_cpu_to_be_32(data_len); 319 klm->pbuf = rte_cpu_to_be_64(addr); 320 klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf); 321 return klm->lkey; 322 323 } 324 325 static __rte_always_inline uint32_t 326 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp, 327 struct rte_crypto_op *op, struct rte_mbuf *mbuf, 328 struct mlx5_wqe_dseg *klm) 329 { 330 uint32_t remain_len = op->sym->cipher.data.length; 331 uint32_t nb_segs = mbuf->nb_segs; 332 uint32_t klm_n = 1u; 333 334 /* First mbuf needs to take the cipher offset. */ 335 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm, 336 op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) { 337 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 338 return 0; 339 } 340 while (remain_len) { 341 nb_segs--; 342 mbuf = mbuf->next; 343 if (unlikely(mbuf == NULL || nb_segs == 0)) { 344 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 345 return 0; 346 } 347 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0, 348 &remain_len) == UINT32_MAX)) { 349 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 350 return 0; 351 } 352 klm_n++; 353 } 354 return klm_n; 355 } 356 357 static __rte_always_inline int 358 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv, 359 struct mlx5_crypto_qp *qp, 360 struct rte_crypto_op *op, 361 struct mlx5_umr_wqe *umr) 362 { 363 struct mlx5_crypto_session *sess = get_sym_session_private_data 364 (op->sym->session, mlx5_crypto_driver_id); 365 struct mlx5_wqe_cseg *cseg = &umr->ctr; 366 struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc; 367 struct mlx5_wqe_dseg *klms = &umr->kseg[0]; 368 struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *) 369 RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1; 370 uint32_t ds; 371 bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src; 372 /* Set UMR WQE. */ 373 uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op, 374 ipl ? op->sym->m_src : op->sym->m_dst, klms); 375 376 if (unlikely(klm_n == 0)) 377 return 0; 378 bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es; 379 if (unlikely(!sess->bsp_res)) { 380 bsf->bsp_res = mlx5_crypto_get_block_size(op); 381 if (unlikely(bsf->bsp_res == UINT32_MAX)) { 382 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 383 return 0; 384 } 385 } else { 386 bsf->bsp_res = sess->bsp_res; 387 } 388 bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length); 389 memcpy(bsf->xts_initial_tweak, 390 rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16); 391 bsf->res_dp = sess->dek_id; 392 mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length); 393 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR); 394 qp->db_pi += priv->umr_wqe_stride; 395 /* Set RDMA_WRITE WQE. */ 396 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 397 klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe)); 398 if (!ipl) { 399 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src, 400 klms); 401 if (unlikely(klm_n == 0)) 402 return 0; 403 } else { 404 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n); 405 } 406 ds = 2 + klm_n; 407 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds); 408 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 409 MLX5_OPCODE_RDMA_WRITE); 410 ds = RTE_ALIGN(ds, 4); 411 qp->db_pi += ds >> 2; 412 /* Set NOP WQE if needed. */ 413 if (priv->max_rdmar_ds > ds) { 414 cseg += ds; 415 ds = priv->max_rdmar_ds - ds; 416 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds); 417 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 418 MLX5_OPCODE_NOP); 419 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */ 420 } 421 qp->wqe = (uint8_t *)cseg; 422 return 1; 423 } 424 425 static __rte_always_inline void 426 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv) 427 { 428 #ifdef RTE_ARCH_64 429 *priv->uar_addr = val; 430 #else /* !RTE_ARCH_64 */ 431 rte_spinlock_lock(&priv->uar32_sl); 432 *(volatile uint32_t *)priv->uar_addr = val; 433 rte_io_wmb(); 434 *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32; 435 rte_spinlock_unlock(&priv->uar32_sl); 436 #endif 437 } 438 439 static uint16_t 440 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 441 uint16_t nb_ops) 442 { 443 struct mlx5_crypto_qp *qp = queue_pair; 444 struct mlx5_crypto_priv *priv = qp->priv; 445 struct mlx5_umr_wqe *umr; 446 struct rte_crypto_op *op; 447 uint16_t mask = qp->entries_n - 1; 448 uint16_t remain = qp->entries_n - (qp->pi - qp->ci); 449 uint32_t idx; 450 451 if (remain < nb_ops) 452 nb_ops = remain; 453 else 454 remain = nb_ops; 455 if (unlikely(remain == 0)) 456 return 0; 457 do { 458 idx = qp->pi & mask; 459 op = *ops++; 460 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf, 461 priv->wqe_set_size * idx); 462 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) { 463 qp->stats.enqueue_err_count++; 464 if (remain != nb_ops) { 465 qp->stats.enqueued_count -= remain; 466 break; 467 } 468 return 0; 469 } 470 qp->ops[idx] = op; 471 qp->pi++; 472 } while (--remain); 473 qp->stats.enqueued_count += nb_ops; 474 rte_io_wmb(); 475 qp->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi); 476 rte_wmb(); 477 mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv); 478 rte_wmb(); 479 return nb_ops; 480 } 481 482 static __rte_noinline void 483 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op) 484 { 485 const uint32_t idx = qp->ci & (qp->entries_n - 1); 486 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *) 487 &qp->cq_obj.cqes[idx]; 488 489 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 490 qp->stats.dequeue_err_count++; 491 DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome)); 492 } 493 494 static uint16_t 495 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 496 uint16_t nb_ops) 497 { 498 struct mlx5_crypto_qp *qp = queue_pair; 499 volatile struct mlx5_cqe *restrict cqe; 500 struct rte_crypto_op *restrict op; 501 const unsigned int cq_size = qp->entries_n; 502 const unsigned int mask = cq_size - 1; 503 uint32_t idx; 504 uint32_t next_idx = qp->ci & mask; 505 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops); 506 uint16_t i = 0; 507 int ret; 508 509 if (unlikely(max == 0)) 510 return 0; 511 do { 512 idx = next_idx; 513 next_idx = (qp->ci + 1) & mask; 514 op = qp->ops[idx]; 515 cqe = &qp->cq_obj.cqes[idx]; 516 ret = check_cqe(cqe, cq_size, qp->ci); 517 rte_io_rmb(); 518 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 519 if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN)) 520 mlx5_crypto_cqe_err_handle(qp, op); 521 break; 522 } 523 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 524 ops[i++] = op; 525 qp->ci++; 526 } while (i < max); 527 if (likely(i != 0)) { 528 rte_io_wmb(); 529 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci); 530 qp->stats.dequeued_count += i; 531 } 532 return i; 533 } 534 535 static void 536 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp) 537 { 538 uint32_t i; 539 540 for (i = 0 ; i < qp->entries_n; i++) { 541 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf, 542 i * priv->wqe_set_size); 543 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *) 544 (cseg + 1); 545 struct mlx5_wqe_umr_bsf_seg *bsf = 546 (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg, 547 priv->umr_wqe_size)) - 1; 548 struct mlx5_wqe_rseg *rseg; 549 550 /* Init UMR WQE. */ 551 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | 552 (priv->umr_wqe_size / MLX5_WSEG_SIZE)); 553 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 554 MLX5_COMP_MODE_OFFSET); 555 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id); 556 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET); 557 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */ 558 ucseg->ko_to_bs = rte_cpu_to_be_32 559 ((RTE_ALIGN(priv->max_segs_num, 4u) << 560 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET)); 561 bsf->keytag = priv->keytag; 562 /* Init RDMA WRITE WQE. */ 563 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 564 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS << 565 MLX5_COMP_MODE_OFFSET) | 566 MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE); 567 rseg = (struct mlx5_wqe_rseg *)(cseg + 1); 568 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id); 569 } 570 } 571 572 static int 573 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, 574 struct mlx5_crypto_qp *qp) 575 { 576 struct mlx5_umr_wqe *umr; 577 uint32_t i; 578 struct mlx5_devx_mkey_attr attr = { 579 .pd = priv->cdev->pdn, 580 .umr_en = 1, 581 .crypto_en = 1, 582 .set_remote_rw = 1, 583 .klm_num = RTE_ALIGN(priv->max_segs_num, 4), 584 }; 585 586 for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0; 587 i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) { 588 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0]; 589 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr); 590 if (!qp->mkey[i]) 591 goto error; 592 } 593 return 0; 594 error: 595 DRV_LOG(ERR, "Failed to allocate indirect mkey."); 596 mlx5_crypto_indirect_mkeys_release(qp, i); 597 return -1; 598 } 599 600 static int 601 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 602 const struct rte_cryptodev_qp_conf *qp_conf, 603 int socket_id) 604 { 605 struct mlx5_crypto_priv *priv = dev->data->dev_private; 606 struct mlx5_devx_qp_attr attr = {0}; 607 struct mlx5_crypto_qp *qp; 608 uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors); 609 uint32_t ret; 610 uint32_t alloc_size = sizeof(*qp); 611 struct mlx5_devx_cq_attr cq_attr = { 612 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar), 613 }; 614 615 if (dev->data->queue_pairs[qp_id] != NULL) 616 mlx5_crypto_queue_pair_release(dev, qp_id); 617 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 618 alloc_size += (sizeof(struct rte_crypto_op *) + 619 sizeof(struct mlx5_devx_obj *)) * 620 RTE_BIT32(log_nb_desc); 621 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 622 socket_id); 623 if (qp == NULL) { 624 DRV_LOG(ERR, "Failed to allocate QP memory."); 625 rte_errno = ENOMEM; 626 return -rte_errno; 627 } 628 if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc, 629 &cq_attr, socket_id) != 0) { 630 DRV_LOG(ERR, "Failed to create CQ."); 631 goto error; 632 } 633 attr.pd = priv->cdev->pdn; 634 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar); 635 attr.cqn = qp->cq_obj.cq->id; 636 attr.rq_size = 0; 637 attr.sq_size = RTE_BIT32(log_nb_desc); 638 attr.ts_format = 639 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format); 640 ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc, 641 &attr, socket_id); 642 if (ret) { 643 DRV_LOG(ERR, "Failed to create QP."); 644 goto error; 645 } 646 if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen, 647 priv->dev_config.socket_id) != 0) { 648 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 649 (uint32_t)qp_id); 650 rte_errno = ENOMEM; 651 goto error; 652 } 653 /* 654 * In Order to configure self loopback, when calling devx qp2rts the 655 * remote QP id that is used is the id of the same QP. 656 */ 657 if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id)) 658 goto error; 659 qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1), 660 RTE_CACHE_LINE_SIZE); 661 qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc)); 662 qp->entries_n = 1 << log_nb_desc; 663 if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) { 664 DRV_LOG(ERR, "Cannot allocate indirect memory regions."); 665 rte_errno = ENOMEM; 666 goto error; 667 } 668 mlx5_crypto_qp_init(priv, qp); 669 qp->priv = priv; 670 dev->data->queue_pairs[qp_id] = qp; 671 return 0; 672 error: 673 mlx5_crypto_qp_release(qp); 674 return -1; 675 } 676 677 static void 678 mlx5_crypto_stats_get(struct rte_cryptodev *dev, 679 struct rte_cryptodev_stats *stats) 680 { 681 int qp_id; 682 683 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 684 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 685 686 stats->enqueued_count += qp->stats.enqueued_count; 687 stats->dequeued_count += qp->stats.dequeued_count; 688 stats->enqueue_err_count += qp->stats.enqueue_err_count; 689 stats->dequeue_err_count += qp->stats.dequeue_err_count; 690 } 691 } 692 693 static void 694 mlx5_crypto_stats_reset(struct rte_cryptodev *dev) 695 { 696 int qp_id; 697 698 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 699 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 700 701 memset(&qp->stats, 0, sizeof(qp->stats)); 702 } 703 } 704 705 static struct rte_cryptodev_ops mlx5_crypto_ops = { 706 .dev_configure = mlx5_crypto_dev_configure, 707 .dev_start = mlx5_crypto_dev_start, 708 .dev_stop = mlx5_crypto_dev_stop, 709 .dev_close = mlx5_crypto_dev_close, 710 .dev_infos_get = mlx5_crypto_dev_infos_get, 711 .stats_get = mlx5_crypto_stats_get, 712 .stats_reset = mlx5_crypto_stats_reset, 713 .queue_pair_setup = mlx5_crypto_queue_pair_setup, 714 .queue_pair_release = mlx5_crypto_queue_pair_release, 715 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 716 .sym_session_configure = mlx5_crypto_sym_session_configure, 717 .sym_session_clear = mlx5_crypto_sym_session_clear, 718 .sym_get_raw_dp_ctx_size = NULL, 719 .sym_configure_raw_dp_ctx = NULL, 720 }; 721 722 static void 723 mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv) 724 { 725 if (priv->uar != NULL) { 726 mlx5_glue->devx_free_uar(priv->uar); 727 priv->uar = NULL; 728 } 729 } 730 731 static int 732 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv) 733 { 734 priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1); 735 if (priv->uar) 736 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar); 737 if (priv->uar == NULL || priv->uar_addr == NULL) { 738 rte_errno = errno; 739 DRV_LOG(ERR, "Failed to allocate UAR."); 740 return -1; 741 } 742 return 0; 743 } 744 745 746 static int 747 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 748 { 749 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 750 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 751 unsigned long tmp; 752 FILE *file; 753 int ret; 754 int i; 755 756 if (strcmp(key, "class") == 0) 757 return 0; 758 if (strcmp(key, "wcs_file") == 0) { 759 file = fopen(val, "rb"); 760 if (file == NULL) { 761 rte_errno = ENOTSUP; 762 return -rte_errno; 763 } 764 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 765 ret = fscanf(file, "%02hhX", &attr->credential[i]); 766 if (ret <= 0) { 767 fclose(file); 768 DRV_LOG(ERR, 769 "Failed to read credential from file."); 770 rte_errno = EINVAL; 771 return -rte_errno; 772 } 773 } 774 fclose(file); 775 devarg_prms->login_devarg = true; 776 return 0; 777 } 778 errno = 0; 779 tmp = strtoul(val, NULL, 0); 780 if (errno) { 781 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 782 return -errno; 783 } 784 if (strcmp(key, "max_segs_num") == 0) { 785 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) { 786 DRV_LOG(WARNING, "Invalid max_segs_num: %d, should" 787 " be less than %d.", 788 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS); 789 rte_errno = EINVAL; 790 return -rte_errno; 791 } 792 devarg_prms->max_segs_num = (uint32_t)tmp; 793 } else if (strcmp(key, "import_kek_id") == 0) { 794 attr->session_import_kek_ptr = (uint32_t)tmp; 795 } else if (strcmp(key, "credential_id") == 0) { 796 attr->credential_pointer = (uint32_t)tmp; 797 } else if (strcmp(key, "keytag") == 0) { 798 devarg_prms->keytag = tmp; 799 } else { 800 DRV_LOG(WARNING, "Invalid key %s.", key); 801 } 802 return 0; 803 } 804 805 static int 806 mlx5_crypto_parse_devargs(struct rte_devargs *devargs, 807 struct mlx5_crypto_devarg_params *devarg_prms) 808 { 809 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 810 struct rte_kvargs *kvlist; 811 812 /* Default values. */ 813 attr->credential_pointer = 0; 814 attr->session_import_kek_ptr = 0; 815 devarg_prms->keytag = 0; 816 devarg_prms->max_segs_num = 8; 817 if (devargs == NULL) { 818 DRV_LOG(ERR, 819 "No login devargs in order to enable crypto operations in the device."); 820 rte_errno = EINVAL; 821 return -1; 822 } 823 kvlist = rte_kvargs_parse(devargs->args, NULL); 824 if (kvlist == NULL) { 825 DRV_LOG(ERR, "Failed to parse devargs."); 826 rte_errno = EINVAL; 827 return -1; 828 } 829 if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler, 830 devarg_prms) != 0) { 831 DRV_LOG(ERR, "Devargs handler function Failed."); 832 rte_kvargs_free(kvlist); 833 rte_errno = EINVAL; 834 return -1; 835 } 836 rte_kvargs_free(kvlist); 837 if (devarg_prms->login_devarg == false) { 838 DRV_LOG(ERR, 839 "No login credential devarg in order to enable crypto operations " 840 "in the device."); 841 rte_errno = EINVAL; 842 return -1; 843 } 844 return 0; 845 } 846 847 static int 848 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev) 849 { 850 struct rte_cryptodev *crypto_dev; 851 struct mlx5_devx_obj *login; 852 struct mlx5_crypto_priv *priv; 853 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 854 struct rte_cryptodev_pmd_init_params init_params = { 855 .name = "", 856 .private_data_size = sizeof(struct mlx5_crypto_priv), 857 .socket_id = cdev->dev->numa_node, 858 .max_nb_queue_pairs = 859 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 860 }; 861 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx); 862 uint16_t rdmw_wqe_size; 863 int ret; 864 865 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 866 DRV_LOG(ERR, "Non-primary process type is not supported."); 867 rte_errno = ENOTSUP; 868 return -rte_errno; 869 } 870 if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) { 871 DRV_LOG(ERR, "Not enough capabilities to support crypto " 872 "operations, maybe old FW/OFED version?"); 873 rte_errno = ENOTSUP; 874 return -ENOTSUP; 875 } 876 ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms); 877 if (ret) { 878 DRV_LOG(ERR, "Failed to parse devargs."); 879 return -rte_errno; 880 } 881 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx, 882 &devarg_prms.login_attr); 883 if (login == NULL) { 884 DRV_LOG(ERR, "Failed to configure login."); 885 return -rte_errno; 886 } 887 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev, 888 &init_params); 889 if (crypto_dev == NULL) { 890 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name); 891 return -ENODEV; 892 } 893 DRV_LOG(INFO, 894 "Crypto device %s was created successfully.", ibdev_name); 895 crypto_dev->dev_ops = &mlx5_crypto_ops; 896 crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst; 897 crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst; 898 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 899 crypto_dev->driver_id = mlx5_crypto_driver_id; 900 priv = crypto_dev->data->dev_private; 901 priv->cdev = cdev; 902 priv->login_obj = login; 903 priv->crypto_dev = crypto_dev; 904 if (mlx5_crypto_uar_prepare(priv) != 0) { 905 rte_cryptodev_pmd_destroy(priv->crypto_dev); 906 return -1; 907 } 908 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 909 priv->max_segs_num = devarg_prms.max_segs_num; 910 priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) + 911 sizeof(struct mlx5_umr_wqe) + 912 RTE_ALIGN(priv->max_segs_num, 4) * 913 sizeof(struct mlx5_wqe_dseg); 914 rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) + 915 sizeof(struct mlx5_wqe_dseg) * 916 (priv->max_segs_num <= 2 ? 2 : 2 + 917 RTE_ALIGN(priv->max_segs_num - 2, 4)); 918 priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size; 919 priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB; 920 priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg); 921 pthread_mutex_lock(&priv_list_lock); 922 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 923 pthread_mutex_unlock(&priv_list_lock); 924 925 rte_cryptodev_pmd_probing_finish(crypto_dev); 926 927 return 0; 928 } 929 930 static int 931 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev) 932 { 933 struct mlx5_crypto_priv *priv = NULL; 934 935 pthread_mutex_lock(&priv_list_lock); 936 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 937 if (priv->crypto_dev->device == cdev->dev) 938 break; 939 if (priv) 940 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 941 pthread_mutex_unlock(&priv_list_lock); 942 if (priv) { 943 mlx5_crypto_uar_release(priv); 944 rte_cryptodev_pmd_destroy(priv->crypto_dev); 945 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 946 } 947 return 0; 948 } 949 950 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 951 { 952 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 953 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 954 }, 955 { 956 .vendor_id = 0 957 } 958 }; 959 960 static struct mlx5_class_driver mlx5_crypto_driver = { 961 .drv_class = MLX5_CLASS_CRYPTO, 962 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 963 .id_table = mlx5_crypto_pci_id_map, 964 .probe = mlx5_crypto_dev_probe, 965 .remove = mlx5_crypto_dev_remove, 966 }; 967 968 RTE_INIT(rte_mlx5_crypto_init) 969 { 970 mlx5_common_init(); 971 if (mlx5_glue != NULL) 972 mlx5_class_driver_register(&mlx5_crypto_driver); 973 } 974 975 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 976 mlx5_crypto_driver_id); 977 978 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 979 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 980 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 981 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 982