1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_errno.h> 8 #include <rte_log.h> 9 #include <rte_pci.h> 10 #include <rte_memory.h> 11 12 #include <mlx5_glue.h> 13 #include <mlx5_common.h> 14 #include <mlx5_common_pci.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_crypto_utils.h" 19 #include "mlx5_crypto.h" 20 21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 23 #define MLX5_CRYPTO_MAX_QPS 1024 24 #define MLX5_CRYPTO_MAX_SEGS 56 25 26 #define MLX5_CRYPTO_FEATURE_FLAGS \ 27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 28 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \ 29 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 30 31 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 32 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 33 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER; 34 35 int mlx5_crypto_logtype; 36 37 uint8_t mlx5_crypto_driver_id; 38 39 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = { 40 { /* AES XTS */ 41 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, 42 {.sym = { 43 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, 44 {.cipher = { 45 .algo = RTE_CRYPTO_CIPHER_AES_XTS, 46 .block_size = 16, 47 .key_size = { 48 .min = 32, 49 .max = 64, 50 .increment = 32 51 }, 52 .iv_size = { 53 .min = 16, 54 .max = 16, 55 .increment = 0 56 }, 57 .dataunit_set = 58 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES | 59 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES, 60 }, } 61 }, } 62 }, 63 }; 64 65 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 66 67 static const struct rte_driver mlx5_drv = { 68 .name = mlx5_crypto_drv_name, 69 .alias = mlx5_crypto_drv_name 70 }; 71 72 static struct cryptodev_driver mlx5_cryptodev_driver; 73 74 struct mlx5_crypto_session { 75 uint32_t bs_bpt_eo_es; 76 /**< bsf_size, bsf_p_type, encryption_order and encryption standard, 77 * saved in big endian format. 78 */ 79 uint32_t bsp_res; 80 /**< crypto_block_size_pointer and reserved 24 bits saved in big 81 * endian format. 82 */ 83 uint32_t iv_offset:16; 84 /**< Starting point for Initialisation Vector. */ 85 struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */ 86 uint32_t dek_id; /**< DEK ID */ 87 } __rte_packed; 88 89 static void 90 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 91 struct rte_cryptodev_info *dev_info) 92 { 93 RTE_SET_USED(dev); 94 if (dev_info != NULL) { 95 dev_info->driver_id = mlx5_crypto_driver_id; 96 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 97 dev_info->capabilities = mlx5_crypto_caps; 98 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 99 dev_info->min_mbuf_headroom_req = 0; 100 dev_info->min_mbuf_tailroom_req = 0; 101 dev_info->sym.max_nb_sessions = 0; 102 /* 103 * If 0, the device does not have any limitation in number of 104 * sessions that can be used. 105 */ 106 } 107 } 108 109 static int 110 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 111 struct rte_cryptodev_config *config) 112 { 113 struct mlx5_crypto_priv *priv = dev->data->dev_private; 114 115 if (config == NULL) { 116 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 117 return -EINVAL; 118 } 119 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 120 DRV_LOG(ERR, 121 "Disabled symmetric crypto feature is not supported."); 122 return -ENOTSUP; 123 } 124 if (mlx5_crypto_dek_setup(priv) != 0) { 125 DRV_LOG(ERR, "Dek hash list creation has failed."); 126 return -ENOMEM; 127 } 128 priv->dev_config = *config; 129 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 130 return 0; 131 } 132 133 static void 134 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 135 { 136 RTE_SET_USED(dev); 137 } 138 139 static int 140 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 141 { 142 RTE_SET_USED(dev); 143 return 0; 144 } 145 146 static int 147 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 148 { 149 struct mlx5_crypto_priv *priv = dev->data->dev_private; 150 151 mlx5_crypto_dek_unset(priv); 152 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 153 return 0; 154 } 155 156 static unsigned int 157 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 158 { 159 return sizeof(struct mlx5_crypto_session); 160 } 161 162 static int 163 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev, 164 struct rte_crypto_sym_xform *xform, 165 struct rte_cryptodev_sym_session *session, 166 struct rte_mempool *mp) 167 { 168 struct mlx5_crypto_priv *priv = dev->data->dev_private; 169 struct mlx5_crypto_session *sess_private_data; 170 struct rte_crypto_cipher_xform *cipher; 171 uint8_t encryption_order; 172 int ret; 173 174 if (unlikely(xform->next != NULL)) { 175 DRV_LOG(ERR, "Xform next is not supported."); 176 return -ENOTSUP; 177 } 178 if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) || 179 (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) { 180 DRV_LOG(ERR, "Only AES-XTS algorithm is supported."); 181 return -ENOTSUP; 182 } 183 ret = rte_mempool_get(mp, (void *)&sess_private_data); 184 if (ret != 0) { 185 DRV_LOG(ERR, 186 "Failed to get session %p private data from mempool.", 187 sess_private_data); 188 return -ENOMEM; 189 } 190 cipher = &xform->cipher; 191 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher); 192 if (sess_private_data->dek == NULL) { 193 rte_mempool_put(mp, sess_private_data); 194 DRV_LOG(ERR, "Failed to prepare dek."); 195 return -ENOMEM; 196 } 197 if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 198 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY; 199 else 200 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE; 201 sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32 202 (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET | 203 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET | 204 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET | 205 MLX5_ENCRYPTION_STANDARD_AES_XTS); 206 switch (xform->cipher.dataunit_len) { 207 case 0: 208 sess_private_data->bsp_res = 0; 209 break; 210 case 512: 211 sess_private_data->bsp_res = rte_cpu_to_be_32 212 ((uint32_t)MLX5_BLOCK_SIZE_512B << 213 MLX5_BLOCK_SIZE_OFFSET); 214 break; 215 case 4096: 216 sess_private_data->bsp_res = rte_cpu_to_be_32 217 ((uint32_t)MLX5_BLOCK_SIZE_4096B << 218 MLX5_BLOCK_SIZE_OFFSET); 219 break; 220 default: 221 DRV_LOG(ERR, "Cipher data unit length is not supported."); 222 return -ENOTSUP; 223 } 224 sess_private_data->iv_offset = cipher->iv.offset; 225 sess_private_data->dek_id = 226 rte_cpu_to_be_32(sess_private_data->dek->obj->id & 227 0xffffff); 228 set_sym_session_private_data(session, dev->driver_id, 229 sess_private_data); 230 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data); 231 return 0; 232 } 233 234 static void 235 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 236 struct rte_cryptodev_sym_session *sess) 237 { 238 struct mlx5_crypto_priv *priv = dev->data->dev_private; 239 struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess, 240 dev->driver_id); 241 242 if (unlikely(spriv == NULL)) { 243 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 244 return; 245 } 246 mlx5_crypto_dek_destroy(priv, spriv->dek); 247 set_sym_session_private_data(sess, dev->driver_id, NULL); 248 rte_mempool_put(rte_mempool_from_obj(spriv), spriv); 249 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 250 } 251 252 static int 253 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id) 254 { 255 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 256 257 if (qp->qp_obj != NULL) 258 claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj)); 259 if (qp->umem_obj != NULL) 260 claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj)); 261 if (qp->umem_buf != NULL) 262 rte_free(qp->umem_buf); 263 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 264 mlx5_devx_cq_destroy(&qp->cq_obj); 265 rte_free(qp); 266 dev->data->queue_pairs[qp_id] = NULL; 267 return 0; 268 } 269 270 static int 271 mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp) 272 { 273 /* 274 * In Order to configure self loopback, when calling these functions the 275 * remote QP id that is used is the id of the same QP. 276 */ 277 if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP, 278 qp->qp_obj->id)) { 279 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).", 280 rte_errno); 281 return -1; 282 } 283 if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP, 284 qp->qp_obj->id)) { 285 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).", 286 rte_errno); 287 return -1; 288 } 289 if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP, 290 qp->qp_obj->id)) { 291 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).", 292 rte_errno); 293 return -1; 294 } 295 return 0; 296 } 297 298 static int 299 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 300 const struct rte_cryptodev_qp_conf *qp_conf, 301 int socket_id) 302 { 303 struct mlx5_crypto_priv *priv = dev->data->dev_private; 304 struct mlx5_devx_qp_attr attr = {0}; 305 struct mlx5_crypto_qp *qp; 306 uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors); 307 uint32_t umem_size = RTE_BIT32(log_nb_desc) * 308 MLX5_CRYPTO_WQE_SET_SIZE + 309 sizeof(*qp->db_rec) * 2; 310 uint32_t alloc_size = sizeof(*qp); 311 struct mlx5_devx_cq_attr cq_attr = { 312 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar), 313 }; 314 315 if (dev->data->queue_pairs[qp_id] != NULL) 316 mlx5_crypto_queue_pair_release(dev, qp_id); 317 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 318 alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc); 319 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 320 socket_id); 321 if (qp == NULL) { 322 DRV_LOG(ERR, "Failed to allocate QP memory."); 323 rte_errno = ENOMEM; 324 return -rte_errno; 325 } 326 if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc, 327 &cq_attr, socket_id) != 0) { 328 DRV_LOG(ERR, "Failed to create CQ."); 329 goto error; 330 } 331 qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id); 332 if (qp->umem_buf == NULL) { 333 DRV_LOG(ERR, "Failed to allocate QP umem."); 334 rte_errno = ENOMEM; 335 goto error; 336 } 337 qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx, 338 (void *)(uintptr_t)qp->umem_buf, 339 umem_size, 340 IBV_ACCESS_LOCAL_WRITE); 341 if (qp->umem_obj == NULL) { 342 DRV_LOG(ERR, "Failed to register QP umem."); 343 goto error; 344 } 345 if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N, 346 priv->dev_config.socket_id) != 0) { 347 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 348 (uint32_t)qp_id); 349 rte_errno = ENOMEM; 350 goto error; 351 } 352 qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen; 353 attr.pd = priv->pdn; 354 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar); 355 attr.cqn = qp->cq_obj.cq->id; 356 attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE)); 357 attr.rq_size = 0; 358 attr.sq_size = RTE_BIT32(log_nb_desc); 359 attr.dbr_umem_valid = 1; 360 attr.wq_umem_id = qp->umem_obj->umem_id; 361 attr.wq_umem_offset = 0; 362 attr.dbr_umem_id = qp->umem_obj->umem_id; 363 attr.dbr_address = RTE_BIT64(log_nb_desc) * 364 MLX5_CRYPTO_WQE_SET_SIZE; 365 qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr); 366 if (qp->qp_obj == NULL) { 367 DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno); 368 goto error; 369 } 370 qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address); 371 if (mlx5_crypto_qp2rts(qp)) 372 goto error; 373 qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1), 374 RTE_CACHE_LINE_SIZE); 375 dev->data->queue_pairs[qp_id] = qp; 376 return 0; 377 error: 378 mlx5_crypto_queue_pair_release(dev, qp_id); 379 return -1; 380 } 381 382 static struct rte_cryptodev_ops mlx5_crypto_ops = { 383 .dev_configure = mlx5_crypto_dev_configure, 384 .dev_start = mlx5_crypto_dev_start, 385 .dev_stop = mlx5_crypto_dev_stop, 386 .dev_close = mlx5_crypto_dev_close, 387 .dev_infos_get = mlx5_crypto_dev_infos_get, 388 .stats_get = NULL, 389 .stats_reset = NULL, 390 .queue_pair_setup = mlx5_crypto_queue_pair_setup, 391 .queue_pair_release = mlx5_crypto_queue_pair_release, 392 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 393 .sym_session_configure = mlx5_crypto_sym_session_configure, 394 .sym_session_clear = mlx5_crypto_sym_session_clear, 395 .sym_get_raw_dp_ctx_size = NULL, 396 .sym_configure_raw_dp_ctx = NULL, 397 }; 398 399 static void 400 mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv) 401 { 402 if (priv->pd != NULL) { 403 claim_zero(mlx5_glue->dealloc_pd(priv->pd)); 404 priv->pd = NULL; 405 } 406 if (priv->uar != NULL) { 407 mlx5_glue->devx_free_uar(priv->uar); 408 priv->uar = NULL; 409 } 410 } 411 412 static int 413 mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv) 414 { 415 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 416 struct mlx5dv_obj obj; 417 struct mlx5dv_pd pd_info; 418 int ret; 419 420 priv->pd = mlx5_glue->alloc_pd(priv->ctx); 421 if (priv->pd == NULL) { 422 DRV_LOG(ERR, "Failed to allocate PD."); 423 return errno ? -errno : -ENOMEM; 424 } 425 obj.pd.in = priv->pd; 426 obj.pd.out = &pd_info; 427 ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD); 428 if (ret != 0) { 429 DRV_LOG(ERR, "Fail to get PD object info."); 430 mlx5_glue->dealloc_pd(priv->pd); 431 priv->pd = NULL; 432 return -errno; 433 } 434 priv->pdn = pd_info.pdn; 435 return 0; 436 #else 437 (void)priv; 438 DRV_LOG(ERR, "Cannot get pdn - no DV support."); 439 return -ENOTSUP; 440 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 441 } 442 443 static int 444 mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv) 445 { 446 if (mlx5_crypto_pd_create(priv) != 0) 447 return -1; 448 priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1); 449 if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) == 450 NULL) { 451 rte_errno = errno; 452 claim_zero(mlx5_glue->dealloc_pd(priv->pd)); 453 DRV_LOG(ERR, "Failed to allocate UAR."); 454 return -1; 455 } 456 return 0; 457 } 458 459 460 static int 461 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 462 { 463 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 464 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 465 unsigned long tmp; 466 FILE *file; 467 int ret; 468 int i; 469 470 if (strcmp(key, "class") == 0) 471 return 0; 472 if (strcmp(key, "wcs_file") == 0) { 473 file = fopen(val, "rb"); 474 if (file == NULL) { 475 rte_errno = ENOTSUP; 476 return -rte_errno; 477 } 478 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 479 ret = fscanf(file, "%02hhX", &attr->credential[i]); 480 if (ret <= 0) { 481 fclose(file); 482 DRV_LOG(ERR, 483 "Failed to read credential from file."); 484 rte_errno = EINVAL; 485 return -rte_errno; 486 } 487 } 488 fclose(file); 489 devarg_prms->login_devarg = true; 490 return 0; 491 } 492 errno = 0; 493 tmp = strtoul(val, NULL, 0); 494 if (errno) { 495 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 496 return -errno; 497 } 498 if (strcmp(key, "max_segs_num") == 0) { 499 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) { 500 DRV_LOG(WARNING, "Invalid max_segs_num: %d, should" 501 " be less than %d.", 502 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS); 503 rte_errno = EINVAL; 504 return -rte_errno; 505 } 506 devarg_prms->max_segs_num = (uint32_t)tmp; 507 } else if (strcmp(key, "import_kek_id") == 0) { 508 attr->session_import_kek_ptr = (uint32_t)tmp; 509 } else if (strcmp(key, "credential_id") == 0) { 510 attr->credential_pointer = (uint32_t)tmp; 511 } else if (strcmp(key, "keytag") == 0) { 512 devarg_prms->keytag = tmp; 513 } else { 514 DRV_LOG(WARNING, "Invalid key %s.", key); 515 } 516 return 0; 517 } 518 519 static int 520 mlx5_crypto_parse_devargs(struct rte_devargs *devargs, 521 struct mlx5_crypto_devarg_params *devarg_prms) 522 { 523 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 524 struct rte_kvargs *kvlist; 525 526 /* Default values. */ 527 attr->credential_pointer = 0; 528 attr->session_import_kek_ptr = 0; 529 devarg_prms->keytag = 0; 530 devarg_prms->max_segs_num = 8; 531 if (devargs == NULL) { 532 DRV_LOG(ERR, 533 "No login devargs in order to enable crypto operations in the device."); 534 rte_errno = EINVAL; 535 return -1; 536 } 537 kvlist = rte_kvargs_parse(devargs->args, NULL); 538 if (kvlist == NULL) { 539 DRV_LOG(ERR, "Failed to parse devargs."); 540 rte_errno = EINVAL; 541 return -1; 542 } 543 if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler, 544 devarg_prms) != 0) { 545 DRV_LOG(ERR, "Devargs handler function Failed."); 546 rte_kvargs_free(kvlist); 547 rte_errno = EINVAL; 548 return -1; 549 } 550 rte_kvargs_free(kvlist); 551 if (devarg_prms->login_devarg == false) { 552 DRV_LOG(ERR, 553 "No login credential devarg in order to enable crypto operations " 554 "in the device."); 555 rte_errno = EINVAL; 556 return -1; 557 } 558 return 0; 559 } 560 561 /** 562 * Callback for memory event. 563 * 564 * @param event_type 565 * Memory event type. 566 * @param addr 567 * Address of memory. 568 * @param len 569 * Size of memory. 570 */ 571 static void 572 mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr, 573 size_t len, void *arg __rte_unused) 574 { 575 struct mlx5_crypto_priv *priv; 576 577 /* Must be called from the primary process. */ 578 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 579 switch (event_type) { 580 case RTE_MEM_EVENT_FREE: 581 pthread_mutex_lock(&priv_list_lock); 582 /* Iterate all the existing mlx5 devices. */ 583 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 584 mlx5_free_mr_by_addr(&priv->mr_scache, 585 priv->ctx->device->name, 586 addr, len); 587 pthread_mutex_unlock(&priv_list_lock); 588 break; 589 case RTE_MEM_EVENT_ALLOC: 590 default: 591 break; 592 } 593 } 594 595 /** 596 * DPDK callback to register a PCI device. 597 * 598 * This function spawns crypto device out of a given PCI device. 599 * 600 * @param[in] pci_drv 601 * PCI driver structure (mlx5_crypto_driver). 602 * @param[in] pci_dev 603 * PCI device information. 604 * 605 * @return 606 * 0 on success, 1 to skip this driver, a negative errno value otherwise 607 * and rte_errno is set. 608 */ 609 static int 610 mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv, 611 struct rte_pci_device *pci_dev) 612 { 613 struct ibv_device *ibv; 614 struct rte_cryptodev *crypto_dev; 615 struct ibv_context *ctx; 616 struct mlx5_devx_obj *login; 617 struct mlx5_crypto_priv *priv; 618 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 619 struct mlx5_hca_attr attr = { 0 }; 620 struct rte_cryptodev_pmd_init_params init_params = { 621 .name = "", 622 .private_data_size = sizeof(struct mlx5_crypto_priv), 623 .socket_id = pci_dev->device.numa_node, 624 .max_nb_queue_pairs = 625 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 626 }; 627 uint16_t rdmw_wqe_size; 628 int ret; 629 630 RTE_SET_USED(pci_drv); 631 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 632 DRV_LOG(ERR, "Non-primary process type is not supported."); 633 rte_errno = ENOTSUP; 634 return -rte_errno; 635 } 636 ibv = mlx5_os_get_ibv_device(&pci_dev->addr); 637 if (ibv == NULL) { 638 DRV_LOG(ERR, "No matching IB device for PCI slot " 639 PCI_PRI_FMT ".", pci_dev->addr.domain, 640 pci_dev->addr.bus, pci_dev->addr.devid, 641 pci_dev->addr.function); 642 return -rte_errno; 643 } 644 DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name); 645 ctx = mlx5_glue->dv_open_device(ibv); 646 if (ctx == NULL) { 647 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name); 648 rte_errno = ENODEV; 649 return -rte_errno; 650 } 651 if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 || 652 attr.crypto == 0 || attr.aes_xts == 0) { 653 DRV_LOG(ERR, "Not enough capabilities to support crypto " 654 "operations, maybe old FW/OFED version?"); 655 claim_zero(mlx5_glue->close_device(ctx)); 656 rte_errno = ENOTSUP; 657 return -ENOTSUP; 658 } 659 ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms); 660 if (ret) { 661 DRV_LOG(ERR, "Failed to parse devargs."); 662 return -rte_errno; 663 } 664 login = mlx5_devx_cmd_create_crypto_login_obj(ctx, 665 &devarg_prms.login_attr); 666 if (login == NULL) { 667 DRV_LOG(ERR, "Failed to configure login."); 668 return -rte_errno; 669 } 670 crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device, 671 &init_params); 672 if (crypto_dev == NULL) { 673 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name); 674 claim_zero(mlx5_glue->close_device(ctx)); 675 return -ENODEV; 676 } 677 DRV_LOG(INFO, 678 "Crypto device %s was created successfully.", ibv->name); 679 crypto_dev->dev_ops = &mlx5_crypto_ops; 680 crypto_dev->dequeue_burst = NULL; 681 crypto_dev->enqueue_burst = NULL; 682 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS; 683 crypto_dev->driver_id = mlx5_crypto_driver_id; 684 priv = crypto_dev->data->dev_private; 685 priv->ctx = ctx; 686 priv->login_obj = login; 687 priv->pci_dev = pci_dev; 688 priv->crypto_dev = crypto_dev; 689 if (mlx5_crypto_hw_global_prepare(priv) != 0) { 690 rte_cryptodev_pmd_destroy(priv->crypto_dev); 691 claim_zero(mlx5_glue->close_device(priv->ctx)); 692 return -1; 693 } 694 if (mlx5_mr_btree_init(&priv->mr_scache.cache, 695 MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) { 696 DRV_LOG(ERR, "Failed to allocate shared cache MR memory."); 697 mlx5_crypto_hw_global_release(priv); 698 rte_cryptodev_pmd_destroy(priv->crypto_dev); 699 claim_zero(mlx5_glue->close_device(priv->ctx)); 700 rte_errno = ENOMEM; 701 return -rte_errno; 702 } 703 priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr; 704 priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr; 705 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 706 priv->max_segs_num = devarg_prms.max_segs_num; 707 priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) + 708 sizeof(struct mlx5_umr_wqe) + 709 RTE_ALIGN(priv->max_segs_num, 4) * 710 sizeof(struct mlx5_wqe_dseg); 711 rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) + 712 sizeof(struct mlx5_wqe_dseg) * 713 (priv->max_segs_num <= 2 ? 2 : 2 + 714 RTE_ALIGN(priv->max_segs_num - 2, 4)); 715 priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size; 716 priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB; 717 priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg); 718 /* Register callback function for global shared MR cache management. */ 719 if (TAILQ_EMPTY(&mlx5_crypto_priv_list)) 720 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", 721 mlx5_crypto_mr_mem_event_cb, 722 NULL); 723 pthread_mutex_lock(&priv_list_lock); 724 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 725 pthread_mutex_unlock(&priv_list_lock); 726 return 0; 727 } 728 729 static int 730 mlx5_crypto_pci_remove(struct rte_pci_device *pdev) 731 { 732 struct mlx5_crypto_priv *priv = NULL; 733 734 pthread_mutex_lock(&priv_list_lock); 735 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 736 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0) 737 break; 738 if (priv) 739 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 740 pthread_mutex_unlock(&priv_list_lock); 741 if (priv) { 742 if (TAILQ_EMPTY(&mlx5_crypto_priv_list)) 743 rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB", 744 NULL); 745 mlx5_mr_release_cache(&priv->mr_scache); 746 mlx5_crypto_hw_global_release(priv); 747 rte_cryptodev_pmd_destroy(priv->crypto_dev); 748 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 749 claim_zero(mlx5_glue->close_device(priv->ctx)); 750 } 751 return 0; 752 } 753 754 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 755 { 756 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 757 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 758 }, 759 { 760 .vendor_id = 0 761 } 762 }; 763 764 static struct mlx5_pci_driver mlx5_crypto_driver = { 765 .driver_class = MLX5_CLASS_CRYPTO, 766 .pci_driver = { 767 .driver = { 768 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 769 }, 770 .id_table = mlx5_crypto_pci_id_map, 771 .probe = mlx5_crypto_pci_probe, 772 .remove = mlx5_crypto_pci_remove, 773 .drv_flags = 0, 774 }, 775 }; 776 777 RTE_INIT(rte_mlx5_crypto_init) 778 { 779 mlx5_common_init(); 780 if (mlx5_glue != NULL) 781 mlx5_pci_driver_register(&mlx5_crypto_driver); 782 } 783 784 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 785 mlx5_crypto_driver_id); 786 787 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 788 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 789 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 790 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 791