1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_eal_paging.h> 8 #include <rte_errno.h> 9 #include <rte_log.h> 10 #include <rte_bus_pci.h> 11 #include <rte_memory.h> 12 13 #include <mlx5_glue.h> 14 #include <mlx5_common.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_crypto_utils.h" 19 #include "mlx5_crypto.h" 20 21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 23 #define MLX5_CRYPTO_MAX_QPS 128 24 #define MLX5_CRYPTO_MAX_SEGS 56 25 26 #define MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode) \ 27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 28 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \ 29 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \ 30 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \ 31 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \ 32 (wrapped_mode ? RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY : 0) | \ 33 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 34 35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 36 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 37 static pthread_mutex_t priv_list_lock; 38 39 int mlx5_crypto_logtype; 40 41 uint8_t mlx5_crypto_driver_id; 42 43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = { 44 { /* AES XTS */ 45 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, 46 {.sym = { 47 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, 48 {.cipher = { 49 .algo = RTE_CRYPTO_CIPHER_AES_XTS, 50 .block_size = 16, 51 .key_size = { 52 .min = 32, 53 .max = 64, 54 .increment = 32 55 }, 56 .iv_size = { 57 .min = 16, 58 .max = 16, 59 .increment = 0 60 }, 61 .dataunit_set = 62 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES | 63 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES | 64 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES, 65 }, } 66 }, } 67 }, 68 }; 69 70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 71 72 static const struct rte_driver mlx5_drv = { 73 .name = mlx5_crypto_drv_name, 74 .alias = mlx5_crypto_drv_name 75 }; 76 77 static struct cryptodev_driver mlx5_cryptodev_driver; 78 79 struct mlx5_crypto_session { 80 uint32_t bs_bpt_eo_es; 81 /**< bsf_size, bsf_p_type, encryption_order and encryption standard, 82 * saved in big endian format. 83 */ 84 uint32_t bsp_res; 85 /**< crypto_block_size_pointer and reserved 24 bits saved in big 86 * endian format. 87 */ 88 uint32_t iv_offset:16; 89 /**< Starting point for Initialisation Vector. */ 90 struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */ 91 uint32_t dek_id; /**< DEK ID */ 92 } __rte_packed; 93 94 static void 95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 96 struct rte_cryptodev_info *dev_info) 97 { 98 struct mlx5_crypto_priv *priv = dev->data->dev_private; 99 100 RTE_SET_USED(dev); 101 if (dev_info != NULL) { 102 dev_info->driver_id = mlx5_crypto_driver_id; 103 dev_info->feature_flags = 104 MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode); 105 dev_info->capabilities = mlx5_crypto_caps; 106 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 107 dev_info->min_mbuf_headroom_req = 0; 108 dev_info->min_mbuf_tailroom_req = 0; 109 dev_info->sym.max_nb_sessions = 0; 110 /* 111 * If 0, the device does not have any limitation in number of 112 * sessions that can be used. 113 */ 114 } 115 } 116 117 static int 118 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 119 struct rte_cryptodev_config *config) 120 { 121 struct mlx5_crypto_priv *priv = dev->data->dev_private; 122 123 if (config == NULL) { 124 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 125 return -EINVAL; 126 } 127 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 128 DRV_LOG(ERR, 129 "Disabled symmetric crypto feature is not supported."); 130 return -ENOTSUP; 131 } 132 if (mlx5_crypto_dek_setup(priv) != 0) { 133 DRV_LOG(ERR, "Dek hash list creation has failed."); 134 return -ENOMEM; 135 } 136 priv->dev_config = *config; 137 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 138 return 0; 139 } 140 141 static void 142 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 143 { 144 RTE_SET_USED(dev); 145 } 146 147 static int 148 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 149 { 150 struct mlx5_crypto_priv *priv = dev->data->dev_private; 151 152 return mlx5_dev_mempool_subscribe(priv->cdev); 153 } 154 155 static int 156 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 157 { 158 struct mlx5_crypto_priv *priv = dev->data->dev_private; 159 160 mlx5_crypto_dek_unset(priv); 161 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 162 return 0; 163 } 164 165 static unsigned int 166 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 167 { 168 return sizeof(struct mlx5_crypto_session); 169 } 170 171 static int 172 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev, 173 struct rte_crypto_sym_xform *xform, 174 struct rte_cryptodev_sym_session *session, 175 struct rte_mempool *mp) 176 { 177 struct mlx5_crypto_priv *priv = dev->data->dev_private; 178 struct mlx5_crypto_session *sess_private_data; 179 struct rte_crypto_cipher_xform *cipher; 180 uint8_t encryption_order; 181 int ret; 182 183 if (unlikely(xform->next != NULL)) { 184 DRV_LOG(ERR, "Xform next is not supported."); 185 return -ENOTSUP; 186 } 187 if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) || 188 (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) { 189 DRV_LOG(ERR, "Only AES-XTS algorithm is supported."); 190 return -ENOTSUP; 191 } 192 ret = rte_mempool_get(mp, (void *)&sess_private_data); 193 if (ret != 0) { 194 DRV_LOG(ERR, 195 "Failed to get session %p private data from mempool.", 196 sess_private_data); 197 return -ENOMEM; 198 } 199 cipher = &xform->cipher; 200 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher); 201 if (sess_private_data->dek == NULL) { 202 rte_mempool_put(mp, sess_private_data); 203 DRV_LOG(ERR, "Failed to prepare dek."); 204 return -ENOMEM; 205 } 206 if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 207 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY; 208 else 209 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE; 210 sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32 211 (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET | 212 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET | 213 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET | 214 MLX5_ENCRYPTION_STANDARD_AES_XTS); 215 switch (xform->cipher.dataunit_len) { 216 case 0: 217 sess_private_data->bsp_res = 0; 218 break; 219 case 512: 220 sess_private_data->bsp_res = rte_cpu_to_be_32 221 ((uint32_t)MLX5_BLOCK_SIZE_512B << 222 MLX5_BLOCK_SIZE_OFFSET); 223 break; 224 case 4096: 225 sess_private_data->bsp_res = rte_cpu_to_be_32 226 ((uint32_t)MLX5_BLOCK_SIZE_4096B << 227 MLX5_BLOCK_SIZE_OFFSET); 228 break; 229 case 1048576: 230 sess_private_data->bsp_res = rte_cpu_to_be_32 231 ((uint32_t)MLX5_BLOCK_SIZE_1MB << 232 MLX5_BLOCK_SIZE_OFFSET); 233 break; 234 default: 235 DRV_LOG(ERR, "Cipher data unit length is not supported."); 236 return -ENOTSUP; 237 } 238 sess_private_data->iv_offset = cipher->iv.offset; 239 sess_private_data->dek_id = 240 rte_cpu_to_be_32(sess_private_data->dek->obj->id & 241 0xffffff); 242 set_sym_session_private_data(session, dev->driver_id, 243 sess_private_data); 244 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data); 245 return 0; 246 } 247 248 static void 249 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 250 struct rte_cryptodev_sym_session *sess) 251 { 252 struct mlx5_crypto_priv *priv = dev->data->dev_private; 253 struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess, 254 dev->driver_id); 255 256 if (unlikely(spriv == NULL)) { 257 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 258 return; 259 } 260 mlx5_crypto_dek_destroy(priv, spriv->dek); 261 set_sym_session_private_data(sess, dev->driver_id, NULL); 262 rte_mempool_put(rte_mempool_from_obj(spriv), spriv); 263 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 264 } 265 266 static void 267 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n) 268 { 269 uint16_t i; 270 271 for (i = 0; i < n; i++) 272 if (qp->mkey[i]) 273 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i])); 274 } 275 276 static void 277 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp) 278 { 279 if (qp == NULL) 280 return; 281 mlx5_devx_qp_destroy(&qp->qp_obj); 282 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 283 mlx5_devx_cq_destroy(&qp->cq_obj); 284 rte_free(qp); 285 } 286 287 static int 288 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id) 289 { 290 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 291 292 mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n); 293 mlx5_crypto_qp_release(qp); 294 dev->data->queue_pairs[qp_id] = NULL; 295 return 0; 296 } 297 298 static __rte_noinline uint32_t 299 mlx5_crypto_get_block_size(struct rte_crypto_op *op) 300 { 301 uint32_t bl = op->sym->cipher.data.length; 302 303 switch (bl) { 304 case (1 << 20): 305 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET); 306 case (1 << 12): 307 return RTE_BE32(MLX5_BLOCK_SIZE_4096B << 308 MLX5_BLOCK_SIZE_OFFSET); 309 case (1 << 9): 310 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET); 311 default: 312 DRV_LOG(ERR, "Unknown block size: %u.", bl); 313 return UINT32_MAX; 314 } 315 } 316 317 static __rte_always_inline uint32_t 318 mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf, 319 struct mlx5_wqe_dseg *klm, uint32_t offset, 320 uint32_t *remain) 321 { 322 uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset); 323 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset); 324 325 if (data_len > *remain) 326 data_len = *remain; 327 *remain -= data_len; 328 klm->bcount = rte_cpu_to_be_32(data_len); 329 klm->pbuf = rte_cpu_to_be_64(addr); 330 klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf); 331 return klm->lkey; 332 333 } 334 335 static __rte_always_inline uint32_t 336 mlx5_crypto_klms_set(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op, 337 struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm) 338 { 339 uint32_t remain_len = op->sym->cipher.data.length; 340 uint32_t nb_segs = mbuf->nb_segs; 341 uint32_t klm_n = 1u; 342 343 /* First mbuf needs to take the cipher offset. */ 344 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, klm, 345 op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) { 346 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 347 return 0; 348 } 349 while (remain_len) { 350 nb_segs--; 351 mbuf = mbuf->next; 352 if (unlikely(mbuf == NULL || nb_segs == 0)) { 353 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 354 return 0; 355 } 356 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0, 357 &remain_len) == UINT32_MAX)) { 358 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 359 return 0; 360 } 361 klm_n++; 362 } 363 return klm_n; 364 } 365 366 static __rte_always_inline int 367 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv, 368 struct mlx5_crypto_qp *qp, 369 struct rte_crypto_op *op, 370 struct mlx5_umr_wqe *umr) 371 { 372 struct mlx5_crypto_session *sess = get_sym_session_private_data 373 (op->sym->session, mlx5_crypto_driver_id); 374 struct mlx5_wqe_cseg *cseg = &umr->ctr; 375 struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc; 376 struct mlx5_wqe_dseg *klms = &umr->kseg[0]; 377 struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *) 378 RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1; 379 uint32_t ds; 380 bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src; 381 /* Set UMR WQE. */ 382 uint32_t klm_n = mlx5_crypto_klms_set(qp, op, 383 ipl ? op->sym->m_src : op->sym->m_dst, klms); 384 385 if (unlikely(klm_n == 0)) 386 return 0; 387 bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es; 388 if (unlikely(!sess->bsp_res)) { 389 bsf->bsp_res = mlx5_crypto_get_block_size(op); 390 if (unlikely(bsf->bsp_res == UINT32_MAX)) { 391 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 392 return 0; 393 } 394 } else { 395 bsf->bsp_res = sess->bsp_res; 396 } 397 bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length); 398 memcpy(bsf->xts_initial_tweak, 399 rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16); 400 bsf->res_dp = sess->dek_id; 401 mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length); 402 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR); 403 qp->db_pi += priv->umr_wqe_stride; 404 /* Set RDMA_WRITE WQE. */ 405 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 406 klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe)); 407 if (!ipl) { 408 klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms); 409 if (unlikely(klm_n == 0)) 410 return 0; 411 } else { 412 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n); 413 } 414 ds = 2 + klm_n; 415 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds); 416 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 417 MLX5_OPCODE_RDMA_WRITE); 418 ds = RTE_ALIGN(ds, 4); 419 qp->db_pi += ds >> 2; 420 /* Set NOP WQE if needed. */ 421 if (priv->max_rdmar_ds > ds) { 422 cseg += ds; 423 ds = priv->max_rdmar_ds - ds; 424 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds); 425 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | 426 MLX5_OPCODE_NOP); 427 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */ 428 } 429 qp->wqe = (uint8_t *)cseg; 430 return 1; 431 } 432 433 static uint16_t 434 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 435 uint16_t nb_ops) 436 { 437 struct mlx5_crypto_qp *qp = queue_pair; 438 struct mlx5_crypto_priv *priv = qp->priv; 439 struct mlx5_umr_wqe *umr; 440 struct rte_crypto_op *op; 441 uint16_t mask = qp->entries_n - 1; 442 uint16_t remain = qp->entries_n - (qp->pi - qp->ci); 443 uint32_t idx; 444 445 if (remain < nb_ops) 446 nb_ops = remain; 447 else 448 remain = nb_ops; 449 if (unlikely(remain == 0)) 450 return 0; 451 do { 452 idx = qp->pi & mask; 453 op = *ops++; 454 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf, 455 priv->wqe_set_size * idx); 456 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) { 457 qp->stats.enqueue_err_count++; 458 if (remain != nb_ops) { 459 qp->stats.enqueued_count -= remain; 460 break; 461 } 462 return 0; 463 } 464 qp->ops[idx] = op; 465 qp->pi++; 466 } while (--remain); 467 qp->stats.enqueued_count += nb_ops; 468 mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->wqe, 469 qp->db_pi, &qp->qp_obj.db_rec[MLX5_SND_DBR], 470 !priv->uar.dbnc); 471 return nb_ops; 472 } 473 474 static __rte_noinline void 475 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op) 476 { 477 const uint32_t idx = qp->ci & (qp->entries_n - 1); 478 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *) 479 &qp->cq_obj.cqes[idx]; 480 481 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 482 qp->stats.dequeue_err_count++; 483 DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome)); 484 } 485 486 static uint16_t 487 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 488 uint16_t nb_ops) 489 { 490 struct mlx5_crypto_qp *qp = queue_pair; 491 volatile struct mlx5_cqe *restrict cqe; 492 struct rte_crypto_op *restrict op; 493 const unsigned int cq_size = qp->entries_n; 494 const unsigned int mask = cq_size - 1; 495 uint32_t idx; 496 uint32_t next_idx = qp->ci & mask; 497 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops); 498 uint16_t i = 0; 499 int ret; 500 501 if (unlikely(max == 0)) 502 return 0; 503 do { 504 idx = next_idx; 505 next_idx = (qp->ci + 1) & mask; 506 op = qp->ops[idx]; 507 cqe = &qp->cq_obj.cqes[idx]; 508 ret = check_cqe(cqe, cq_size, qp->ci); 509 rte_io_rmb(); 510 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 511 if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN)) 512 mlx5_crypto_cqe_err_handle(qp, op); 513 break; 514 } 515 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 516 ops[i++] = op; 517 qp->ci++; 518 } while (i < max); 519 if (likely(i != 0)) { 520 rte_io_wmb(); 521 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci); 522 qp->stats.dequeued_count += i; 523 } 524 return i; 525 } 526 527 static void 528 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp) 529 { 530 uint32_t i; 531 532 for (i = 0 ; i < qp->entries_n; i++) { 533 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf, 534 i * priv->wqe_set_size); 535 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *) 536 (cseg + 1); 537 struct mlx5_wqe_umr_bsf_seg *bsf = 538 (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg, 539 priv->umr_wqe_size)) - 1; 540 struct mlx5_wqe_rseg *rseg; 541 542 /* Init UMR WQE. */ 543 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | 544 (priv->umr_wqe_size / MLX5_WSEG_SIZE)); 545 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 546 MLX5_COMP_MODE_OFFSET); 547 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id); 548 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET); 549 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */ 550 ucseg->ko_to_bs = rte_cpu_to_be_32 551 ((MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size) << 552 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET)); 553 bsf->keytag = priv->keytag; 554 /* Init RDMA WRITE WQE. */ 555 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size); 556 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS << 557 MLX5_COMP_MODE_OFFSET) | 558 MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE); 559 rseg = (struct mlx5_wqe_rseg *)(cseg + 1); 560 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id); 561 } 562 } 563 564 static int 565 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, 566 struct mlx5_crypto_qp *qp) 567 { 568 struct mlx5_umr_wqe *umr; 569 uint32_t i; 570 struct mlx5_devx_mkey_attr attr = { 571 .pd = priv->cdev->pdn, 572 .umr_en = 1, 573 .crypto_en = 1, 574 .set_remote_rw = 1, 575 .klm_num = MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size), 576 }; 577 578 for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0; 579 i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) { 580 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0]; 581 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr); 582 if (!qp->mkey[i]) 583 goto error; 584 } 585 return 0; 586 error: 587 DRV_LOG(ERR, "Failed to allocate indirect mkey."); 588 mlx5_crypto_indirect_mkeys_release(qp, i); 589 return -1; 590 } 591 592 static int 593 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 594 const struct rte_cryptodev_qp_conf *qp_conf, 595 int socket_id) 596 { 597 struct mlx5_crypto_priv *priv = dev->data->dev_private; 598 struct mlx5_devx_qp_attr attr = {0}; 599 struct mlx5_crypto_qp *qp; 600 uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors); 601 uint32_t ret; 602 uint32_t alloc_size = sizeof(*qp); 603 uint32_t log_wqbb_n; 604 struct mlx5_devx_cq_attr cq_attr = { 605 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar.obj), 606 }; 607 608 if (dev->data->queue_pairs[qp_id] != NULL) 609 mlx5_crypto_queue_pair_release(dev, qp_id); 610 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 611 alloc_size += (sizeof(struct rte_crypto_op *) + 612 sizeof(struct mlx5_devx_obj *)) * 613 RTE_BIT32(log_nb_desc); 614 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 615 socket_id); 616 if (qp == NULL) { 617 DRV_LOG(ERR, "Failed to allocate QP memory."); 618 rte_errno = ENOMEM; 619 return -rte_errno; 620 } 621 if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc, 622 &cq_attr, socket_id) != 0) { 623 DRV_LOG(ERR, "Failed to create CQ."); 624 goto error; 625 } 626 log_wqbb_n = rte_log2_u32(RTE_BIT32(log_nb_desc) * 627 (priv->wqe_set_size / MLX5_SEND_WQE_BB)); 628 attr.pd = priv->cdev->pdn; 629 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj); 630 attr.cqn = qp->cq_obj.cq->id; 631 attr.num_of_receive_wqes = 0; 632 attr.num_of_send_wqbbs = RTE_BIT32(log_wqbb_n); 633 attr.ts_format = 634 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format); 635 ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, 636 attr.num_of_send_wqbbs * MLX5_WQE_SIZE, 637 &attr, socket_id); 638 if (ret) { 639 DRV_LOG(ERR, "Failed to create QP."); 640 goto error; 641 } 642 if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen, 643 priv->dev_config.socket_id) != 0) { 644 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 645 (uint32_t)qp_id); 646 rte_errno = ENOMEM; 647 goto error; 648 } 649 /* 650 * In Order to configure self loopback, when calling devx qp2rts the 651 * remote QP id that is used is the id of the same QP. 652 */ 653 if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id)) 654 goto error; 655 qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1), 656 RTE_CACHE_LINE_SIZE); 657 qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc)); 658 qp->entries_n = 1 << log_nb_desc; 659 if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) { 660 DRV_LOG(ERR, "Cannot allocate indirect memory regions."); 661 rte_errno = ENOMEM; 662 goto error; 663 } 664 mlx5_crypto_qp_init(priv, qp); 665 qp->priv = priv; 666 dev->data->queue_pairs[qp_id] = qp; 667 return 0; 668 error: 669 mlx5_crypto_qp_release(qp); 670 return -1; 671 } 672 673 static void 674 mlx5_crypto_stats_get(struct rte_cryptodev *dev, 675 struct rte_cryptodev_stats *stats) 676 { 677 int qp_id; 678 679 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 680 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 681 682 stats->enqueued_count += qp->stats.enqueued_count; 683 stats->dequeued_count += qp->stats.dequeued_count; 684 stats->enqueue_err_count += qp->stats.enqueue_err_count; 685 stats->dequeue_err_count += qp->stats.dequeue_err_count; 686 } 687 } 688 689 static void 690 mlx5_crypto_stats_reset(struct rte_cryptodev *dev) 691 { 692 int qp_id; 693 694 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 695 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 696 697 memset(&qp->stats, 0, sizeof(qp->stats)); 698 } 699 } 700 701 static struct rte_cryptodev_ops mlx5_crypto_ops = { 702 .dev_configure = mlx5_crypto_dev_configure, 703 .dev_start = mlx5_crypto_dev_start, 704 .dev_stop = mlx5_crypto_dev_stop, 705 .dev_close = mlx5_crypto_dev_close, 706 .dev_infos_get = mlx5_crypto_dev_infos_get, 707 .stats_get = mlx5_crypto_stats_get, 708 .stats_reset = mlx5_crypto_stats_reset, 709 .queue_pair_setup = mlx5_crypto_queue_pair_setup, 710 .queue_pair_release = mlx5_crypto_queue_pair_release, 711 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 712 .sym_session_configure = mlx5_crypto_sym_session_configure, 713 .sym_session_clear = mlx5_crypto_sym_session_clear, 714 .sym_get_raw_dp_ctx_size = NULL, 715 .sym_configure_raw_dp_ctx = NULL, 716 }; 717 718 static int 719 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 720 { 721 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 722 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 723 unsigned long tmp; 724 FILE *file; 725 int ret; 726 int i; 727 728 if (strcmp(key, "wcs_file") == 0) { 729 file = fopen(val, "rb"); 730 if (file == NULL) { 731 rte_errno = ENOTSUP; 732 return -rte_errno; 733 } 734 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 735 ret = fscanf(file, "%02hhX", &attr->credential[i]); 736 if (ret <= 0) { 737 fclose(file); 738 DRV_LOG(ERR, 739 "Failed to read credential from file."); 740 rte_errno = EINVAL; 741 return -rte_errno; 742 } 743 } 744 fclose(file); 745 devarg_prms->login_devarg = true; 746 return 0; 747 } 748 errno = 0; 749 tmp = strtoul(val, NULL, 0); 750 if (errno) { 751 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 752 return -errno; 753 } 754 if (strcmp(key, "max_segs_num") == 0) { 755 if (!tmp) { 756 DRV_LOG(ERR, "max_segs_num must be greater than 0."); 757 rte_errno = EINVAL; 758 return -rte_errno; 759 } 760 devarg_prms->max_segs_num = (uint32_t)tmp; 761 } else if (strcmp(key, "import_kek_id") == 0) { 762 attr->session_import_kek_ptr = (uint32_t)tmp; 763 } else if (strcmp(key, "credential_id") == 0) { 764 attr->credential_pointer = (uint32_t)tmp; 765 } else if (strcmp(key, "keytag") == 0) { 766 devarg_prms->keytag = tmp; 767 } 768 return 0; 769 } 770 771 static int 772 mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist, 773 struct mlx5_crypto_devarg_params *devarg_prms, 774 bool wrapped_mode) 775 { 776 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 777 const char **params = (const char *[]){ 778 "credential_id", 779 "import_kek_id", 780 "keytag", 781 "max_segs_num", 782 "wcs_file", 783 NULL, 784 }; 785 786 /* Default values. */ 787 attr->credential_pointer = 0; 788 attr->session_import_kek_ptr = 0; 789 devarg_prms->keytag = 0; 790 devarg_prms->max_segs_num = 8; 791 if (mkvlist == NULL) { 792 if (!wrapped_mode) 793 return 0; 794 DRV_LOG(ERR, 795 "No login devargs in order to enable crypto operations in the device."); 796 rte_errno = EINVAL; 797 return -1; 798 } 799 if (mlx5_kvargs_process(mkvlist, params, mlx5_crypto_args_check_handler, 800 devarg_prms) != 0) { 801 DRV_LOG(ERR, "Devargs handler function Failed."); 802 rte_errno = EINVAL; 803 return -1; 804 } 805 if (devarg_prms->login_devarg == false && wrapped_mode) { 806 DRV_LOG(ERR, 807 "No login credential devarg in order to enable crypto operations in the device while in wrapped import method."); 808 rte_errno = EINVAL; 809 return -1; 810 } 811 return 0; 812 } 813 814 /* 815 * Calculate UMR WQE size and RDMA Write WQE size with the 816 * following limitations: 817 * - Each WQE size is multiple of 64. 818 * - The summarize of both UMR WQE and RDMA_W WQE is a power of 2. 819 * - The number of entries in the UMR WQE's KLM list is multiple of 4. 820 */ 821 static void 822 mlx5_crypto_get_wqe_sizes(uint32_t segs_num, uint32_t *umr_size, 823 uint32_t *rdmaw_size) 824 { 825 uint32_t diff, wqe_set_size; 826 827 *umr_size = MLX5_CRYPTO_UMR_WQE_STATIC_SIZE + 828 RTE_ALIGN(segs_num, 4) * 829 sizeof(struct mlx5_wqe_dseg); 830 /* Make sure UMR WQE size is multiple of WQBB. */ 831 *umr_size = RTE_ALIGN(*umr_size, MLX5_SEND_WQE_BB); 832 *rdmaw_size = sizeof(struct mlx5_rdma_write_wqe) + 833 sizeof(struct mlx5_wqe_dseg) * 834 (segs_num <= 2 ? 2 : 2 + 835 RTE_ALIGN(segs_num - 2, 4)); 836 /* Make sure RDMA_WRITE WQE size is multiple of WQBB. */ 837 *rdmaw_size = RTE_ALIGN(*rdmaw_size, MLX5_SEND_WQE_BB); 838 wqe_set_size = *rdmaw_size + *umr_size; 839 diff = rte_align32pow2(wqe_set_size) - wqe_set_size; 840 /* Make sure wqe_set size is power of 2. */ 841 if (diff) 842 *umr_size += diff; 843 } 844 845 static uint8_t 846 mlx5_crypto_max_segs_num(uint16_t max_wqe_size) 847 { 848 int klms_sizes = max_wqe_size - MLX5_CRYPTO_UMR_WQE_STATIC_SIZE; 849 uint32_t max_segs_cap = RTE_ALIGN_FLOOR(klms_sizes, MLX5_SEND_WQE_BB) / 850 sizeof(struct mlx5_wqe_dseg); 851 852 MLX5_ASSERT(klms_sizes >= MLX5_SEND_WQE_BB); 853 while (max_segs_cap) { 854 uint32_t umr_wqe_size, rdmw_wqe_size; 855 856 mlx5_crypto_get_wqe_sizes(max_segs_cap, &umr_wqe_size, 857 &rdmw_wqe_size); 858 if (umr_wqe_size <= max_wqe_size && 859 rdmw_wqe_size <= max_wqe_size) 860 break; 861 max_segs_cap -= 4; 862 } 863 return max_segs_cap; 864 } 865 866 static int 867 mlx5_crypto_configure_wqe_size(struct mlx5_crypto_priv *priv, 868 uint16_t max_wqe_size, uint32_t max_segs_num) 869 { 870 uint32_t rdmw_wqe_size, umr_wqe_size; 871 872 mlx5_crypto_get_wqe_sizes(max_segs_num, &umr_wqe_size, 873 &rdmw_wqe_size); 874 priv->wqe_set_size = rdmw_wqe_size + umr_wqe_size; 875 if (umr_wqe_size > max_wqe_size || 876 rdmw_wqe_size > max_wqe_size) { 877 DRV_LOG(ERR, "Invalid max_segs_num: %u. should be %u or lower.", 878 max_segs_num, 879 mlx5_crypto_max_segs_num(max_wqe_size)); 880 rte_errno = EINVAL; 881 return -EINVAL; 882 } 883 priv->umr_wqe_size = (uint16_t)umr_wqe_size; 884 priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB; 885 priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg); 886 return 0; 887 } 888 889 static int 890 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev, 891 struct mlx5_kvargs_ctrl *mkvlist) 892 { 893 struct rte_cryptodev *crypto_dev; 894 struct mlx5_devx_obj *login; 895 struct mlx5_crypto_priv *priv; 896 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 897 struct rte_cryptodev_pmd_init_params init_params = { 898 .name = "", 899 .private_data_size = sizeof(struct mlx5_crypto_priv), 900 .socket_id = cdev->dev->numa_node, 901 .max_nb_queue_pairs = 902 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 903 }; 904 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx); 905 int ret; 906 bool wrapped_mode; 907 908 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 909 DRV_LOG(ERR, "Non-primary process type is not supported."); 910 rte_errno = ENOTSUP; 911 return -rte_errno; 912 } 913 if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) { 914 DRV_LOG(ERR, "Not enough capabilities to support crypto " 915 "operations, maybe old FW/OFED version?"); 916 rte_errno = ENOTSUP; 917 return -ENOTSUP; 918 } 919 wrapped_mode = !!cdev->config.hca_attr.crypto_wrapped_import_method; 920 ret = mlx5_crypto_parse_devargs(mkvlist, &devarg_prms, wrapped_mode); 921 if (ret) { 922 DRV_LOG(ERR, "Failed to parse devargs."); 923 return -rte_errno; 924 } 925 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev, 926 &init_params); 927 if (crypto_dev == NULL) { 928 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name); 929 return -ENODEV; 930 } 931 DRV_LOG(INFO, 932 "Crypto device %s was created successfully.", ibdev_name); 933 crypto_dev->dev_ops = &mlx5_crypto_ops; 934 crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst; 935 crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst; 936 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode); 937 crypto_dev->driver_id = mlx5_crypto_driver_id; 938 priv = crypto_dev->data->dev_private; 939 priv->cdev = cdev; 940 priv->crypto_dev = crypto_dev; 941 priv->is_wrapped_mode = wrapped_mode; 942 if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) { 943 rte_cryptodev_pmd_destroy(priv->crypto_dev); 944 return -1; 945 } 946 if (wrapped_mode) { 947 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx, 948 &devarg_prms.login_attr); 949 if (login == NULL) { 950 DRV_LOG(ERR, "Failed to configure login."); 951 mlx5_devx_uar_release(&priv->uar); 952 rte_cryptodev_pmd_destroy(priv->crypto_dev); 953 return -rte_errno; 954 } 955 priv->login_obj = login; 956 } 957 ret = mlx5_crypto_configure_wqe_size(priv, 958 cdev->config.hca_attr.max_wqe_sz_sq, devarg_prms.max_segs_num); 959 if (ret) { 960 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 961 mlx5_devx_uar_release(&priv->uar); 962 rte_cryptodev_pmd_destroy(priv->crypto_dev); 963 return -1; 964 } 965 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 966 DRV_LOG(INFO, "Max number of segments: %u.", 967 (unsigned int)RTE_MIN( 968 MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size), 969 (uint16_t)(priv->max_rdmar_ds - 2))); 970 pthread_mutex_lock(&priv_list_lock); 971 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 972 pthread_mutex_unlock(&priv_list_lock); 973 974 rte_cryptodev_pmd_probing_finish(crypto_dev); 975 976 return 0; 977 } 978 979 static int 980 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev) 981 { 982 struct mlx5_crypto_priv *priv = NULL; 983 984 pthread_mutex_lock(&priv_list_lock); 985 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 986 if (priv->crypto_dev->device == cdev->dev) 987 break; 988 if (priv) 989 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 990 pthread_mutex_unlock(&priv_list_lock); 991 if (priv) { 992 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 993 mlx5_devx_uar_release(&priv->uar); 994 rte_cryptodev_pmd_destroy(priv->crypto_dev); 995 } 996 return 0; 997 } 998 999 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 1000 { 1001 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1002 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 1003 }, 1004 { 1005 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1006 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 1007 }, 1008 { 1009 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1010 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 1011 }, 1012 { 1013 .vendor_id = 0 1014 } 1015 }; 1016 1017 static struct mlx5_class_driver mlx5_crypto_driver = { 1018 .drv_class = MLX5_CLASS_CRYPTO, 1019 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 1020 .id_table = mlx5_crypto_pci_id_map, 1021 .probe = mlx5_crypto_dev_probe, 1022 .remove = mlx5_crypto_dev_remove, 1023 }; 1024 1025 RTE_INIT(rte_mlx5_crypto_init) 1026 { 1027 pthread_mutex_init(&priv_list_lock, NULL); 1028 mlx5_common_init(); 1029 if (mlx5_glue != NULL) 1030 mlx5_class_driver_register(&mlx5_crypto_driver); 1031 } 1032 1033 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 1034 mlx5_crypto_driver_id); 1035 1036 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 1037 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 1038 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 1039 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 1040