1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_eal_paging.h> 8 #include <rte_errno.h> 9 #include <rte_log.h> 10 #include <bus_pci_driver.h> 11 #include <rte_memory.h> 12 13 #include <mlx5_glue.h> 14 #include <mlx5_common.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_crypto_utils.h" 19 #include "mlx5_crypto.h" 20 21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 23 #define MLX5_CRYPTO_MAX_QPS 128 24 #define MLX5_CRYPTO_MAX_SEGS 56 25 26 #define MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode) \ 27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 28 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \ 29 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \ 30 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \ 31 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \ 32 (wrapped_mode ? RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY : 0) | \ 33 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 34 35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 36 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 37 static pthread_mutex_t priv_list_lock; 38 39 int mlx5_crypto_logtype; 40 41 uint8_t mlx5_crypto_driver_id; 42 43 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 44 45 static const struct rte_driver mlx5_drv = { 46 .name = mlx5_crypto_drv_name, 47 .alias = mlx5_crypto_drv_name 48 }; 49 50 static struct cryptodev_driver mlx5_cryptodev_driver; 51 52 static void 53 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 54 struct rte_cryptodev_info *dev_info) 55 { 56 struct mlx5_crypto_priv *priv = dev->data->dev_private; 57 58 RTE_SET_USED(dev); 59 if (dev_info != NULL) { 60 dev_info->driver_id = mlx5_crypto_driver_id; 61 dev_info->feature_flags = 62 MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode); 63 dev_info->capabilities = priv->caps; 64 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 65 if (priv->caps->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 66 dev_info->min_mbuf_headroom_req = MLX5_CRYPTO_GCM_MAX_AAD; 67 dev_info->min_mbuf_tailroom_req = MLX5_CRYPTO_GCM_MAX_DIGEST; 68 } else { 69 dev_info->min_mbuf_headroom_req = 0; 70 dev_info->min_mbuf_tailroom_req = 0; 71 } 72 dev_info->sym.max_nb_sessions = 0; 73 /* 74 * If 0, the device does not have any limitation in number of 75 * sessions that can be used. 76 */ 77 } 78 } 79 80 void 81 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, 82 uint16_t n) 83 { 84 uint32_t i; 85 86 for (i = 0; i < n; i++) 87 if (qp->mkey[i]) 88 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i])); 89 } 90 91 int 92 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, 93 struct mlx5_crypto_qp *qp, 94 struct mlx5_devx_mkey_attr *attr, 95 mlx5_crypto_mkey_update_t update_cb) 96 { 97 uint32_t i; 98 99 for (i = 0; i < qp->entries_n; i++) { 100 attr->klm_array = update_cb(priv, qp, i); 101 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, attr); 102 if (!qp->mkey[i]) 103 goto error; 104 } 105 return 0; 106 error: 107 DRV_LOG(ERR, "Failed to allocate indirect mkey."); 108 mlx5_crypto_indirect_mkeys_release(qp, i); 109 return -1; 110 } 111 112 static int 113 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 114 struct rte_cryptodev_config *config) 115 { 116 struct mlx5_crypto_priv *priv = dev->data->dev_private; 117 118 if (config == NULL) { 119 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 120 return -EINVAL; 121 } 122 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 123 DRV_LOG(ERR, 124 "Disabled symmetric crypto feature is not supported."); 125 return -ENOTSUP; 126 } 127 if (mlx5_crypto_dek_setup(priv) != 0) { 128 DRV_LOG(ERR, "Dek hash list creation has failed."); 129 return -ENOMEM; 130 } 131 priv->dev_config = *config; 132 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 133 return 0; 134 } 135 136 static void 137 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 138 { 139 RTE_SET_USED(dev); 140 } 141 142 static int 143 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 144 { 145 struct mlx5_crypto_priv *priv = dev->data->dev_private; 146 147 return mlx5_dev_mempool_subscribe(priv->cdev); 148 } 149 150 static int 151 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 152 { 153 struct mlx5_crypto_priv *priv = dev->data->dev_private; 154 155 mlx5_crypto_dek_unset(priv); 156 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 157 return 0; 158 } 159 160 static unsigned int 161 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 162 { 163 return sizeof(struct mlx5_crypto_session); 164 } 165 166 static void 167 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 168 struct rte_cryptodev_sym_session *sess) 169 { 170 struct mlx5_crypto_priv *priv = dev->data->dev_private; 171 struct mlx5_crypto_session *spriv = CRYPTODEV_GET_SYM_SESS_PRIV(sess); 172 173 if (unlikely(spriv == NULL)) { 174 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 175 return; 176 } 177 mlx5_crypto_dek_destroy(priv, spriv->dek); 178 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 179 } 180 181 static void 182 mlx5_crypto_stats_get(struct rte_cryptodev *dev, 183 struct rte_cryptodev_stats *stats) 184 { 185 int qp_id; 186 187 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 188 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 189 190 stats->enqueued_count += qp->stats.enqueued_count; 191 stats->dequeued_count += qp->stats.dequeued_count; 192 stats->enqueue_err_count += qp->stats.enqueue_err_count; 193 stats->dequeue_err_count += qp->stats.dequeue_err_count; 194 } 195 } 196 197 static void 198 mlx5_crypto_stats_reset(struct rte_cryptodev *dev) 199 { 200 int qp_id; 201 202 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 203 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 204 205 memset(&qp->stats, 0, sizeof(qp->stats)); 206 } 207 } 208 209 static struct rte_cryptodev_ops mlx5_crypto_ops = { 210 .dev_configure = mlx5_crypto_dev_configure, 211 .dev_start = mlx5_crypto_dev_start, 212 .dev_stop = mlx5_crypto_dev_stop, 213 .dev_close = mlx5_crypto_dev_close, 214 .dev_infos_get = mlx5_crypto_dev_infos_get, 215 .stats_get = mlx5_crypto_stats_get, 216 .stats_reset = mlx5_crypto_stats_reset, 217 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 218 .sym_session_clear = mlx5_crypto_sym_session_clear, 219 .sym_get_raw_dp_ctx_size = NULL, 220 .sym_configure_raw_dp_ctx = NULL, 221 }; 222 223 static int 224 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 225 { 226 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 227 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 228 unsigned long tmp; 229 FILE *file; 230 int ret; 231 int i; 232 233 if (strcmp(key, "wcs_file") == 0) { 234 file = fopen(val, "rb"); 235 if (file == NULL) { 236 rte_errno = ENOTSUP; 237 return -rte_errno; 238 } 239 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 240 ret = fscanf(file, "%02hhX", &attr->credential[i]); 241 if (ret <= 0) { 242 fclose(file); 243 DRV_LOG(ERR, 244 "Failed to read credential from file."); 245 rte_errno = EINVAL; 246 return -rte_errno; 247 } 248 } 249 fclose(file); 250 devarg_prms->login_devarg = true; 251 return 0; 252 } 253 errno = 0; 254 tmp = strtoul(val, NULL, 0); 255 if (errno) { 256 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 257 return -errno; 258 } 259 if (strcmp(key, "max_segs_num") == 0) { 260 if (!tmp) { 261 DRV_LOG(ERR, "max_segs_num must be greater than 0."); 262 rte_errno = EINVAL; 263 return -rte_errno; 264 } 265 devarg_prms->max_segs_num = (uint32_t)tmp; 266 } else if (strcmp(key, "import_kek_id") == 0) { 267 attr->session_import_kek_ptr = (uint32_t)tmp; 268 } else if (strcmp(key, "credential_id") == 0) { 269 attr->credential_pointer = (uint32_t)tmp; 270 } else if (strcmp(key, "keytag") == 0) { 271 devarg_prms->keytag = tmp; 272 } else if (strcmp(key, "algo") == 0) { 273 if (tmp == 1) { 274 devarg_prms->is_aes_gcm = 1; 275 } else if (tmp > 1) { 276 DRV_LOG(ERR, "Invalid algo."); 277 rte_errno = EINVAL; 278 return -rte_errno; 279 } 280 } 281 return 0; 282 } 283 284 static int 285 mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist, 286 struct mlx5_crypto_devarg_params *devarg_prms, 287 bool wrapped_mode) 288 { 289 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 290 const char **params = (const char *[]){ 291 "credential_id", 292 "import_kek_id", 293 "keytag", 294 "max_segs_num", 295 "wcs_file", 296 "algo", 297 NULL, 298 }; 299 300 /* Default values. */ 301 attr->credential_pointer = 0; 302 attr->session_import_kek_ptr = 0; 303 devarg_prms->keytag = 0; 304 devarg_prms->max_segs_num = 8; 305 if (mkvlist == NULL) { 306 if (!wrapped_mode) 307 return 0; 308 DRV_LOG(ERR, 309 "No login devargs in order to enable crypto operations in the device."); 310 rte_errno = EINVAL; 311 return -1; 312 } 313 if (mlx5_kvargs_process(mkvlist, params, mlx5_crypto_args_check_handler, 314 devarg_prms) != 0) { 315 DRV_LOG(ERR, "Devargs handler function Failed."); 316 rte_errno = EINVAL; 317 return -1; 318 } 319 if (devarg_prms->login_devarg == false && wrapped_mode) { 320 DRV_LOG(ERR, 321 "No login credential devarg in order to enable crypto operations in the device while in wrapped import method."); 322 rte_errno = EINVAL; 323 return -1; 324 } 325 return 0; 326 } 327 328 static int 329 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev, 330 struct mlx5_kvargs_ctrl *mkvlist) 331 { 332 struct rte_cryptodev *crypto_dev; 333 struct mlx5_devx_obj *login; 334 struct mlx5_crypto_priv *priv; 335 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 336 struct rte_cryptodev_pmd_init_params init_params = { 337 .name = "", 338 .private_data_size = sizeof(struct mlx5_crypto_priv), 339 .socket_id = cdev->dev->numa_node, 340 .max_nb_queue_pairs = 341 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 342 }; 343 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx); 344 int ret; 345 bool wrapped_mode; 346 347 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 348 DRV_LOG(ERR, "Non-primary process type is not supported."); 349 rte_errno = ENOTSUP; 350 return -rte_errno; 351 } 352 if (!cdev->config.hca_attr.crypto || 353 (!cdev->config.hca_attr.aes_xts && 354 !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) { 355 DRV_LOG(ERR, "Not enough capabilities to support crypto " 356 "operations, maybe old FW/OFED version?"); 357 rte_errno = ENOTSUP; 358 return -ENOTSUP; 359 } 360 wrapped_mode = !!cdev->config.hca_attr.crypto_wrapped_import_method; 361 ret = mlx5_crypto_parse_devargs(mkvlist, &devarg_prms, wrapped_mode); 362 if (ret) { 363 DRV_LOG(ERR, "Failed to parse devargs."); 364 return -rte_errno; 365 } 366 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev, 367 &init_params); 368 if (crypto_dev == NULL) { 369 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name); 370 return -ENODEV; 371 } 372 DRV_LOG(INFO, 373 "Crypto device %s was created successfully.", ibdev_name); 374 crypto_dev->dev_ops = &mlx5_crypto_ops; 375 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode); 376 crypto_dev->driver_id = mlx5_crypto_driver_id; 377 priv = crypto_dev->data->dev_private; 378 priv->cdev = cdev; 379 priv->crypto_dev = crypto_dev; 380 priv->is_wrapped_mode = wrapped_mode; 381 priv->max_segs_num = devarg_prms.max_segs_num; 382 /* Init and override AES-GCM configuration. */ 383 if (devarg_prms.is_aes_gcm) { 384 ret = mlx5_crypto_gcm_init(priv); 385 if (ret) { 386 rte_cryptodev_pmd_destroy(priv->crypto_dev); 387 DRV_LOG(ERR, "Failed to init AES-GCM crypto."); 388 return -ENOTSUP; 389 } 390 } else { 391 ret = mlx5_crypto_xts_init(priv); 392 if (ret) { 393 rte_cryptodev_pmd_destroy(priv->crypto_dev); 394 DRV_LOG(ERR, "Failed to init AES-XTS crypto."); 395 return -ENOTSUP; 396 } 397 } 398 if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) { 399 rte_cryptodev_pmd_destroy(priv->crypto_dev); 400 return -1; 401 } 402 if (wrapped_mode) { 403 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx, 404 &devarg_prms.login_attr); 405 if (login == NULL) { 406 DRV_LOG(ERR, "Failed to configure login."); 407 mlx5_devx_uar_release(&priv->uar); 408 rte_cryptodev_pmd_destroy(priv->crypto_dev); 409 return -rte_errno; 410 } 411 priv->login_obj = login; 412 } 413 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 414 DRV_LOG(INFO, "Max number of segments: %u.", 415 (unsigned int)RTE_MIN( 416 MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size), 417 (uint16_t)(priv->max_rdmar_ds - 2))); 418 pthread_mutex_lock(&priv_list_lock); 419 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 420 pthread_mutex_unlock(&priv_list_lock); 421 422 rte_cryptodev_pmd_probing_finish(crypto_dev); 423 424 return 0; 425 } 426 427 static int 428 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev) 429 { 430 struct mlx5_crypto_priv *priv = NULL; 431 432 pthread_mutex_lock(&priv_list_lock); 433 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 434 if (priv->crypto_dev->device == cdev->dev) 435 break; 436 if (priv) 437 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 438 pthread_mutex_unlock(&priv_list_lock); 439 if (priv) { 440 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 441 mlx5_devx_uar_release(&priv->uar); 442 rte_cryptodev_pmd_destroy(priv->crypto_dev); 443 } 444 return 0; 445 } 446 447 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 448 { 449 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 450 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 451 }, 452 { 453 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 454 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 455 }, 456 { 457 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 458 PCI_DEVICE_ID_MELLANOX_BLUEFIELD2) 459 }, 460 { 461 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 462 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 463 }, 464 { 465 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 466 PCI_DEVICE_ID_MELLANOX_BLUEFIELD3) 467 }, 468 { 469 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 470 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 471 }, 472 { 473 .vendor_id = 0 474 } 475 }; 476 477 static struct mlx5_class_driver mlx5_crypto_driver = { 478 .drv_class = MLX5_CLASS_CRYPTO, 479 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 480 .id_table = mlx5_crypto_pci_id_map, 481 .probe = mlx5_crypto_dev_probe, 482 .remove = mlx5_crypto_dev_remove, 483 }; 484 485 RTE_INIT(rte_mlx5_crypto_init) 486 { 487 pthread_mutex_init(&priv_list_lock, NULL); 488 mlx5_common_init(); 489 if (mlx5_glue != NULL) 490 mlx5_class_driver_register(&mlx5_crypto_driver); 491 } 492 493 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 494 mlx5_crypto_driver_id); 495 496 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 497 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 498 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 499 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 500