1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mempool.h> 7 #include <rte_eal_paging.h> 8 #include <rte_errno.h> 9 #include <rte_log.h> 10 #include <bus_pci_driver.h> 11 #include <rte_memory.h> 12 13 #include <mlx5_glue.h> 14 #include <mlx5_common.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_crypto_utils.h" 19 #include "mlx5_crypto.h" 20 21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5 22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5 23 #define MLX5_CRYPTO_MAX_QPS 128 24 #define MLX5_CRYPTO_MAX_SEGS 56 25 26 #define MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode) \ 27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \ 28 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \ 29 (wrapped_mode ? RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY : 0) | \ 30 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS) 31 32 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list = 33 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list); 34 static pthread_mutex_t priv_list_lock; 35 36 int mlx5_crypto_logtype; 37 38 uint8_t mlx5_crypto_driver_id; 39 40 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME); 41 42 static const struct rte_driver mlx5_drv = { 43 .name = mlx5_crypto_drv_name, 44 .alias = mlx5_crypto_drv_name 45 }; 46 47 static struct cryptodev_driver mlx5_cryptodev_driver; 48 49 static void 50 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev, 51 struct rte_cryptodev_info *dev_info) 52 { 53 struct mlx5_crypto_priv *priv = dev->data->dev_private; 54 55 RTE_SET_USED(dev); 56 if (dev_info != NULL) { 57 dev_info->driver_id = mlx5_crypto_driver_id; 58 dev_info->feature_flags = 59 MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode); 60 if (!mlx5_crypto_is_ipsec_opt(priv)) 61 dev_info->feature_flags |= 62 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 63 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | 64 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 65 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT; 66 67 dev_info->capabilities = priv->caps; 68 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS; 69 if (priv->caps->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 70 dev_info->min_mbuf_headroom_req = MLX5_CRYPTO_GCM_MAX_AAD; 71 dev_info->min_mbuf_tailroom_req = MLX5_CRYPTO_GCM_MAX_DIGEST; 72 } else { 73 dev_info->min_mbuf_headroom_req = 0; 74 dev_info->min_mbuf_tailroom_req = 0; 75 } 76 dev_info->sym.max_nb_sessions = 0; 77 /* 78 * If 0, the device does not have any limitation in number of 79 * sessions that can be used. 80 */ 81 } 82 } 83 84 void 85 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, 86 uint16_t n) 87 { 88 uint32_t i; 89 90 for (i = 0; i < n; i++) 91 if (qp->mkey[i]) 92 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i])); 93 } 94 95 int 96 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, 97 struct mlx5_crypto_qp *qp, 98 struct mlx5_devx_mkey_attr *attr, 99 mlx5_crypto_mkey_update_t update_cb) 100 { 101 uint32_t i; 102 103 for (i = 0; i < qp->entries_n; i++) { 104 attr->klm_array = update_cb(priv, qp, i); 105 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, attr); 106 if (!qp->mkey[i]) 107 goto error; 108 } 109 return 0; 110 error: 111 DRV_LOG(ERR, "Failed to allocate indirect mkey."); 112 mlx5_crypto_indirect_mkeys_release(qp, i); 113 return -1; 114 } 115 116 static int 117 mlx5_crypto_dev_configure(struct rte_cryptodev *dev, 118 struct rte_cryptodev_config *config) 119 { 120 struct mlx5_crypto_priv *priv = dev->data->dev_private; 121 122 if (config == NULL) { 123 DRV_LOG(ERR, "Invalid crypto dev configure parameters."); 124 return -EINVAL; 125 } 126 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) { 127 DRV_LOG(ERR, 128 "Disabled symmetric crypto feature is not supported."); 129 return -ENOTSUP; 130 } 131 if (mlx5_crypto_dek_setup(priv) != 0) { 132 DRV_LOG(ERR, "Dek hash list creation has failed."); 133 return -ENOMEM; 134 } 135 priv->dev_config = *config; 136 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id); 137 return 0; 138 } 139 140 static void 141 mlx5_crypto_dev_stop(struct rte_cryptodev *dev) 142 { 143 RTE_SET_USED(dev); 144 } 145 146 static int 147 mlx5_crypto_dev_start(struct rte_cryptodev *dev) 148 { 149 struct mlx5_crypto_priv *priv = dev->data->dev_private; 150 151 return mlx5_dev_mempool_subscribe(priv->cdev); 152 } 153 154 static int 155 mlx5_crypto_dev_close(struct rte_cryptodev *dev) 156 { 157 struct mlx5_crypto_priv *priv = dev->data->dev_private; 158 159 mlx5_crypto_dek_unset(priv); 160 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id); 161 return 0; 162 } 163 164 static unsigned int 165 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 166 { 167 return sizeof(struct mlx5_crypto_session); 168 } 169 170 static void 171 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev, 172 struct rte_cryptodev_sym_session *sess) 173 { 174 struct mlx5_crypto_priv *priv = dev->data->dev_private; 175 struct mlx5_crypto_session *spriv = CRYPTODEV_GET_SYM_SESS_PRIV(sess); 176 177 if (unlikely(spriv == NULL)) { 178 DRV_LOG(ERR, "Failed to get session %p private data.", spriv); 179 return; 180 } 181 mlx5_crypto_dek_destroy(priv, spriv->dek); 182 DRV_LOG(DEBUG, "Session %p was cleared.", spriv); 183 } 184 185 static void 186 mlx5_crypto_stats_get(struct rte_cryptodev *dev, 187 struct rte_cryptodev_stats *stats) 188 { 189 int qp_id; 190 191 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 192 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 193 194 stats->enqueued_count += qp->stats.enqueued_count; 195 stats->dequeued_count += qp->stats.dequeued_count; 196 stats->enqueue_err_count += qp->stats.enqueue_err_count; 197 stats->dequeue_err_count += qp->stats.dequeue_err_count; 198 } 199 } 200 201 static void 202 mlx5_crypto_stats_reset(struct rte_cryptodev *dev) 203 { 204 int qp_id; 205 206 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 207 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id]; 208 209 memset(&qp->stats, 0, sizeof(qp->stats)); 210 } 211 } 212 213 static struct rte_cryptodev_ops mlx5_crypto_ops = { 214 .dev_configure = mlx5_crypto_dev_configure, 215 .dev_start = mlx5_crypto_dev_start, 216 .dev_stop = mlx5_crypto_dev_stop, 217 .dev_close = mlx5_crypto_dev_close, 218 .dev_infos_get = mlx5_crypto_dev_infos_get, 219 .stats_get = mlx5_crypto_stats_get, 220 .stats_reset = mlx5_crypto_stats_reset, 221 .sym_session_get_size = mlx5_crypto_sym_session_get_size, 222 .sym_session_clear = mlx5_crypto_sym_session_clear, 223 .sym_get_raw_dp_ctx_size = NULL, 224 .sym_configure_raw_dp_ctx = NULL, 225 }; 226 227 static int 228 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque) 229 { 230 struct mlx5_crypto_devarg_params *devarg_prms = opaque; 231 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 232 unsigned long tmp; 233 FILE *file; 234 int ret; 235 int i; 236 237 if (strcmp(key, "wcs_file") == 0) { 238 file = fopen(val, "rb"); 239 if (file == NULL) { 240 rte_errno = ENOTSUP; 241 return -rte_errno; 242 } 243 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) { 244 ret = fscanf(file, "%02hhX", &attr->credential[i]); 245 if (ret <= 0) { 246 fclose(file); 247 DRV_LOG(ERR, 248 "Failed to read credential from file."); 249 rte_errno = EINVAL; 250 return -rte_errno; 251 } 252 } 253 fclose(file); 254 devarg_prms->login_devarg = true; 255 return 0; 256 } else if (strcmp(key, "crypto_mode") == 0) { 257 if (strcmp(val, "full_capable") == 0) { 258 devarg_prms->crypto_mode = MLX5_CRYPTO_FULL_CAPABLE; 259 } else if (strcmp(val, "ipsec_opt") == 0) { 260 devarg_prms->crypto_mode = MLX5_CRYPTO_IPSEC_OPT; 261 } else { 262 DRV_LOG(ERR, "Invalid crypto mode: %s", val); 263 rte_errno = EINVAL; 264 return -rte_errno; 265 } 266 } 267 errno = 0; 268 tmp = strtoul(val, NULL, 0); 269 if (errno) { 270 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val); 271 return -errno; 272 } 273 if (strcmp(key, "max_segs_num") == 0) { 274 if (!tmp) { 275 DRV_LOG(ERR, "max_segs_num must be greater than 0."); 276 rte_errno = EINVAL; 277 return -rte_errno; 278 } 279 devarg_prms->max_segs_num = (uint32_t)tmp; 280 } else if (strcmp(key, "import_kek_id") == 0) { 281 attr->session_import_kek_ptr = (uint32_t)tmp; 282 } else if (strcmp(key, "credential_id") == 0) { 283 attr->credential_pointer = (uint32_t)tmp; 284 } else if (strcmp(key, "keytag") == 0) { 285 devarg_prms->keytag = tmp; 286 } else if (strcmp(key, "algo") == 0) { 287 if (tmp == 1) { 288 devarg_prms->is_aes_gcm = 1; 289 } else if (tmp > 1) { 290 DRV_LOG(ERR, "Invalid algo."); 291 rte_errno = EINVAL; 292 return -rte_errno; 293 } 294 } 295 return 0; 296 } 297 298 static int 299 mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist, 300 struct mlx5_crypto_devarg_params *devarg_prms, 301 bool wrapped_mode) 302 { 303 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr; 304 const char **params = (const char *[]){ 305 "credential_id", 306 "import_kek_id", 307 "keytag", 308 "max_segs_num", 309 "wcs_file", 310 "algo", 311 "crypto_mode", 312 NULL, 313 }; 314 315 /* Default values. */ 316 attr->credential_pointer = 0; 317 attr->session_import_kek_ptr = 0; 318 devarg_prms->keytag = 0; 319 devarg_prms->max_segs_num = 8; 320 if (mkvlist == NULL) { 321 if (!wrapped_mode) 322 return 0; 323 DRV_LOG(ERR, 324 "No login devargs in order to enable crypto operations in the device."); 325 rte_errno = EINVAL; 326 return -1; 327 } 328 if (mlx5_kvargs_process(mkvlist, params, mlx5_crypto_args_check_handler, 329 devarg_prms) != 0) { 330 DRV_LOG(ERR, "Devargs handler function Failed."); 331 rte_errno = EINVAL; 332 return -1; 333 } 334 if (devarg_prms->login_devarg == false && wrapped_mode) { 335 DRV_LOG(ERR, 336 "No login credential devarg in order to enable crypto operations in the device while in wrapped import method."); 337 rte_errno = EINVAL; 338 return -1; 339 } 340 return 0; 341 } 342 343 static int 344 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev, 345 struct mlx5_kvargs_ctrl *mkvlist) 346 { 347 struct rte_cryptodev *crypto_dev; 348 struct mlx5_devx_obj *login; 349 struct mlx5_crypto_priv *priv; 350 struct mlx5_crypto_devarg_params devarg_prms = { 0 }; 351 struct rte_cryptodev_pmd_init_params init_params = { 352 .name = "", 353 .private_data_size = sizeof(struct mlx5_crypto_priv), 354 .socket_id = cdev->dev->numa_node, 355 .max_nb_queue_pairs = 356 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 357 }; 358 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx); 359 int ret; 360 bool wrapped_mode; 361 362 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 363 DRV_LOG(ERR, "Non-primary process type is not supported."); 364 rte_errno = ENOTSUP; 365 return -rte_errno; 366 } 367 if (!cdev->config.hca_attr.crypto || 368 (!cdev->config.hca_attr.aes_xts && 369 !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) { 370 DRV_LOG(ERR, "Not enough capabilities to support crypto " 371 "operations, maybe old FW/OFED version?"); 372 rte_errno = ENOTSUP; 373 return -ENOTSUP; 374 } 375 wrapped_mode = !!cdev->config.hca_attr.crypto_wrapped_import_method; 376 ret = mlx5_crypto_parse_devargs(mkvlist, &devarg_prms, wrapped_mode); 377 if (ret) { 378 DRV_LOG(ERR, "Failed to parse devargs."); 379 return -rte_errno; 380 } 381 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev, 382 &init_params); 383 if (crypto_dev == NULL) { 384 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name); 385 return -ENODEV; 386 } 387 DRV_LOG(INFO, 388 "Crypto device %s was created successfully.", ibdev_name); 389 crypto_dev->dev_ops = &mlx5_crypto_ops; 390 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode); 391 crypto_dev->driver_id = mlx5_crypto_driver_id; 392 priv = crypto_dev->data->dev_private; 393 priv->cdev = cdev; 394 priv->crypto_dev = crypto_dev; 395 priv->is_wrapped_mode = wrapped_mode; 396 priv->max_segs_num = devarg_prms.max_segs_num; 397 priv->crypto_mode = devarg_prms.crypto_mode; 398 /* Init and override AES-GCM configuration. */ 399 if (devarg_prms.is_aes_gcm) { 400 ret = mlx5_crypto_gcm_init(priv); 401 if (ret) { 402 rte_cryptodev_pmd_destroy(priv->crypto_dev); 403 DRV_LOG(ERR, "Failed to init AES-GCM crypto."); 404 return -ENOTSUP; 405 } 406 } else { 407 ret = mlx5_crypto_xts_init(priv); 408 if (ret) { 409 rte_cryptodev_pmd_destroy(priv->crypto_dev); 410 DRV_LOG(ERR, "Failed to init AES-XTS crypto."); 411 return -ENOTSUP; 412 } 413 } 414 if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) { 415 rte_cryptodev_pmd_destroy(priv->crypto_dev); 416 return -1; 417 } 418 if (wrapped_mode) { 419 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx, 420 &devarg_prms.login_attr); 421 if (login == NULL) { 422 DRV_LOG(ERR, "Failed to configure login."); 423 mlx5_devx_uar_release(&priv->uar); 424 rte_cryptodev_pmd_destroy(priv->crypto_dev); 425 return -rte_errno; 426 } 427 priv->login_obj = login; 428 } 429 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag); 430 DRV_LOG(INFO, "Max number of segments: %u.", 431 (unsigned int)RTE_MIN( 432 MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size), 433 (uint16_t)(priv->max_rdmar_ds - 2))); 434 pthread_mutex_lock(&priv_list_lock); 435 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next); 436 pthread_mutex_unlock(&priv_list_lock); 437 438 rte_cryptodev_pmd_probing_finish(crypto_dev); 439 440 return 0; 441 } 442 443 static int 444 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev) 445 { 446 struct mlx5_crypto_priv *priv = NULL; 447 448 pthread_mutex_lock(&priv_list_lock); 449 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next) 450 if (priv->crypto_dev->device == cdev->dev) 451 break; 452 if (priv) 453 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next); 454 pthread_mutex_unlock(&priv_list_lock); 455 if (priv) { 456 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj)); 457 mlx5_devx_uar_release(&priv->uar); 458 rte_cryptodev_pmd_destroy(priv->crypto_dev); 459 } 460 return 0; 461 } 462 463 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = { 464 { 465 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 466 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 467 }, 468 { 469 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 470 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 471 }, 472 { 473 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 474 PCI_DEVICE_ID_MELLANOX_BLUEFIELD2) 475 }, 476 { 477 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 478 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 479 }, 480 { 481 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 482 PCI_DEVICE_ID_MELLANOX_BLUEFIELD3) 483 }, 484 { 485 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 486 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 487 }, 488 { 489 .vendor_id = 0 490 } 491 }; 492 493 static struct mlx5_class_driver mlx5_crypto_driver = { 494 .drv_class = MLX5_CLASS_CRYPTO, 495 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME), 496 .id_table = mlx5_crypto_pci_id_map, 497 .probe = mlx5_crypto_dev_probe, 498 .remove = mlx5_crypto_dev_remove, 499 }; 500 501 RTE_INIT(rte_mlx5_crypto_init) 502 { 503 pthread_mutex_init(&priv_list_lock, NULL); 504 mlx5_common_init(); 505 if (mlx5_glue != NULL) 506 mlx5_class_driver_register(&mlx5_crypto_driver); 507 } 508 509 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv, 510 mlx5_crypto_driver_id); 511 512 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE) 513 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__); 514 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map); 515 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 516