1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_mempool.h>
7 #include <rte_eal_paging.h>
8 #include <rte_errno.h>
9 #include <rte_log.h>
10 #include <bus_pci_driver.h>
11 #include <rte_memory.h>
12
13 #include <mlx5_glue.h>
14 #include <mlx5_common.h>
15 #include <mlx5_devx_cmds.h>
16 #include <mlx5_common_os.h>
17
18 #include "mlx5_crypto_utils.h"
19 #include "mlx5_crypto.h"
20
21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
23 #define MLX5_CRYPTO_MAX_QPS 128
24 #define MLX5_CRYPTO_MAX_SEGS 56
25
26 #define MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode) \
27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
28 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
29 (wrapped_mode ? RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY : 0) | \
30 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
31
32 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
33 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
34 static pthread_mutex_t priv_list_lock;
35
36 int mlx5_crypto_logtype;
37
38 uint8_t mlx5_crypto_driver_id;
39
40 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
41
42 static const struct rte_driver mlx5_drv = {
43 .name = mlx5_crypto_drv_name,
44 .alias = mlx5_crypto_drv_name
45 };
46
47 static struct cryptodev_driver mlx5_cryptodev_driver;
48
49 static void
mlx5_crypto_dev_infos_get(struct rte_cryptodev * dev,struct rte_cryptodev_info * dev_info)50 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
51 struct rte_cryptodev_info *dev_info)
52 {
53 struct mlx5_crypto_priv *priv = dev->data->dev_private;
54
55 RTE_SET_USED(dev);
56 if (dev_info != NULL) {
57 dev_info->driver_id = mlx5_crypto_driver_id;
58 dev_info->feature_flags =
59 MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode);
60 if (!mlx5_crypto_is_ipsec_opt(priv))
61 dev_info->feature_flags |=
62 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
63 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
64 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
65 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT;
66
67 dev_info->capabilities = priv->caps;
68 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
69 if (priv->caps->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
70 dev_info->min_mbuf_headroom_req = MLX5_CRYPTO_GCM_MAX_AAD;
71 dev_info->min_mbuf_tailroom_req = MLX5_CRYPTO_GCM_MAX_DIGEST;
72 } else {
73 dev_info->min_mbuf_headroom_req = 0;
74 dev_info->min_mbuf_tailroom_req = 0;
75 }
76 dev_info->sym.max_nb_sessions = 0;
77 /*
78 * If 0, the device does not have any limitation in number of
79 * sessions that can be used.
80 */
81 }
82 }
83
84 void
mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp * qp,uint16_t n)85 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp,
86 uint16_t n)
87 {
88 uint32_t i;
89
90 for (i = 0; i < n; i++)
91 if (qp->mkey[i])
92 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
93 }
94
95 int
mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv * priv,struct mlx5_crypto_qp * qp,struct mlx5_devx_mkey_attr * attr,mlx5_crypto_mkey_update_t update_cb)96 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
97 struct mlx5_crypto_qp *qp,
98 struct mlx5_devx_mkey_attr *attr,
99 mlx5_crypto_mkey_update_t update_cb)
100 {
101 uint32_t i;
102
103 for (i = 0; i < qp->entries_n; i++) {
104 attr->klm_array = update_cb(priv, qp, i);
105 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, attr);
106 if (!qp->mkey[i])
107 goto error;
108 }
109 return 0;
110 error:
111 DRV_LOG(ERR, "Failed to allocate indirect mkey.");
112 mlx5_crypto_indirect_mkeys_release(qp, i);
113 return -1;
114 }
115
116 static int
mlx5_crypto_dev_configure(struct rte_cryptodev * dev,struct rte_cryptodev_config * config)117 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
118 struct rte_cryptodev_config *config)
119 {
120 struct mlx5_crypto_priv *priv = dev->data->dev_private;
121
122 if (config == NULL) {
123 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
124 return -EINVAL;
125 }
126 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
127 DRV_LOG(ERR,
128 "Disabled symmetric crypto feature is not supported.");
129 return -ENOTSUP;
130 }
131 if (mlx5_crypto_dek_setup(priv) != 0) {
132 DRV_LOG(ERR, "Dek hash list creation has failed.");
133 return -ENOMEM;
134 }
135 priv->dev_config = *config;
136 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
137 return 0;
138 }
139
140 static void
mlx5_crypto_dev_stop(struct rte_cryptodev * dev)141 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
142 {
143 RTE_SET_USED(dev);
144 }
145
146 static int
mlx5_crypto_dev_start(struct rte_cryptodev * dev)147 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
148 {
149 struct mlx5_crypto_priv *priv = dev->data->dev_private;
150
151 return mlx5_dev_mempool_subscribe(priv->cdev);
152 }
153
154 static int
mlx5_crypto_dev_close(struct rte_cryptodev * dev)155 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
156 {
157 struct mlx5_crypto_priv *priv = dev->data->dev_private;
158
159 mlx5_crypto_dek_unset(priv);
160 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
161 return 0;
162 }
163
164 static unsigned int
mlx5_crypto_sym_session_get_size(struct rte_cryptodev * dev __rte_unused)165 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
166 {
167 return sizeof(struct mlx5_crypto_session);
168 }
169
170 static void
mlx5_crypto_sym_session_clear(struct rte_cryptodev * dev,struct rte_cryptodev_sym_session * sess)171 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
172 struct rte_cryptodev_sym_session *sess)
173 {
174 struct mlx5_crypto_priv *priv = dev->data->dev_private;
175 struct mlx5_crypto_session *spriv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
176
177 if (unlikely(spriv == NULL)) {
178 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
179 return;
180 }
181 mlx5_crypto_dek_destroy(priv, spriv->dek);
182 DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
183 }
184
185 static void
mlx5_crypto_stats_get(struct rte_cryptodev * dev,struct rte_cryptodev_stats * stats)186 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
187 struct rte_cryptodev_stats *stats)
188 {
189 int qp_id;
190
191 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
192 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
193
194 stats->enqueued_count += qp->stats.enqueued_count;
195 stats->dequeued_count += qp->stats.dequeued_count;
196 stats->enqueue_err_count += qp->stats.enqueue_err_count;
197 stats->dequeue_err_count += qp->stats.dequeue_err_count;
198 }
199 }
200
201 static void
mlx5_crypto_stats_reset(struct rte_cryptodev * dev)202 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
203 {
204 int qp_id;
205
206 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
207 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
208
209 memset(&qp->stats, 0, sizeof(qp->stats));
210 }
211 }
212
213 static struct rte_cryptodev_ops mlx5_crypto_ops = {
214 .dev_configure = mlx5_crypto_dev_configure,
215 .dev_start = mlx5_crypto_dev_start,
216 .dev_stop = mlx5_crypto_dev_stop,
217 .dev_close = mlx5_crypto_dev_close,
218 .dev_infos_get = mlx5_crypto_dev_infos_get,
219 .stats_get = mlx5_crypto_stats_get,
220 .stats_reset = mlx5_crypto_stats_reset,
221 .sym_session_get_size = mlx5_crypto_sym_session_get_size,
222 .sym_session_clear = mlx5_crypto_sym_session_clear,
223 .sym_get_raw_dp_ctx_size = NULL,
224 .sym_configure_raw_dp_ctx = NULL,
225 };
226
227 static int
mlx5_crypto_args_check_handler(const char * key,const char * val,void * opaque)228 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
229 {
230 struct mlx5_crypto_devarg_params *devarg_prms = opaque;
231 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
232 unsigned long tmp;
233 FILE *file;
234 int ret;
235 int i;
236
237 if (strcmp(key, "wcs_file") == 0) {
238 file = fopen(val, "rb");
239 if (file == NULL) {
240 rte_errno = ENOTSUP;
241 return -rte_errno;
242 }
243 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
244 ret = fscanf(file, "%02hhX", &attr->credential[i]);
245 if (ret <= 0) {
246 fclose(file);
247 DRV_LOG(ERR,
248 "Failed to read credential from file.");
249 rte_errno = EINVAL;
250 return -rte_errno;
251 }
252 }
253 fclose(file);
254 devarg_prms->login_devarg = true;
255 return 0;
256 } else if (strcmp(key, "crypto_mode") == 0) {
257 if (strcmp(val, "full_capable") == 0) {
258 devarg_prms->crypto_mode = MLX5_CRYPTO_FULL_CAPABLE;
259 } else if (strcmp(val, "ipsec_opt") == 0) {
260 devarg_prms->crypto_mode = MLX5_CRYPTO_IPSEC_OPT;
261 } else {
262 DRV_LOG(ERR, "Invalid crypto mode: %s", val);
263 rte_errno = EINVAL;
264 return -rte_errno;
265 }
266 }
267 errno = 0;
268 tmp = strtoul(val, NULL, 0);
269 if (errno) {
270 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
271 return -errno;
272 }
273 if (strcmp(key, "max_segs_num") == 0) {
274 if (!tmp) {
275 DRV_LOG(ERR, "max_segs_num must be greater than 0.");
276 rte_errno = EINVAL;
277 return -rte_errno;
278 }
279 devarg_prms->max_segs_num = (uint32_t)tmp;
280 } else if (strcmp(key, "import_kek_id") == 0) {
281 attr->session_import_kek_ptr = (uint32_t)tmp;
282 } else if (strcmp(key, "credential_id") == 0) {
283 attr->credential_pointer = (uint32_t)tmp;
284 } else if (strcmp(key, "keytag") == 0) {
285 devarg_prms->keytag = tmp;
286 } else if (strcmp(key, "algo") == 0) {
287 if (tmp == 1) {
288 devarg_prms->is_aes_gcm = 1;
289 } else if (tmp > 1) {
290 DRV_LOG(ERR, "Invalid algo.");
291 rte_errno = EINVAL;
292 return -rte_errno;
293 }
294 }
295 return 0;
296 }
297
298 static int
mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl * mkvlist,struct mlx5_crypto_devarg_params * devarg_prms,bool wrapped_mode)299 mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
300 struct mlx5_crypto_devarg_params *devarg_prms,
301 bool wrapped_mode)
302 {
303 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
304 const char **params = (const char *[]){
305 "credential_id",
306 "import_kek_id",
307 "keytag",
308 "max_segs_num",
309 "wcs_file",
310 "algo",
311 "crypto_mode",
312 NULL,
313 };
314
315 /* Default values. */
316 attr->credential_pointer = 0;
317 attr->session_import_kek_ptr = 0;
318 devarg_prms->keytag = 0;
319 devarg_prms->max_segs_num = 8;
320 if (mkvlist == NULL) {
321 if (!wrapped_mode)
322 return 0;
323 DRV_LOG(ERR,
324 "No login devargs in order to enable crypto operations in the device.");
325 rte_errno = EINVAL;
326 return -1;
327 }
328 if (mlx5_kvargs_process(mkvlist, params, mlx5_crypto_args_check_handler,
329 devarg_prms) != 0) {
330 DRV_LOG(ERR, "Devargs handler function Failed.");
331 rte_errno = EINVAL;
332 return -1;
333 }
334 if (devarg_prms->login_devarg == false && wrapped_mode) {
335 DRV_LOG(ERR,
336 "No login credential devarg in order to enable crypto operations in the device while in wrapped import method.");
337 rte_errno = EINVAL;
338 return -1;
339 }
340 return 0;
341 }
342
343 static int
mlx5_crypto_dev_probe(struct mlx5_common_device * cdev,struct mlx5_kvargs_ctrl * mkvlist)344 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
345 struct mlx5_kvargs_ctrl *mkvlist)
346 {
347 struct rte_cryptodev *crypto_dev;
348 struct mlx5_devx_obj *login;
349 struct mlx5_crypto_priv *priv;
350 struct mlx5_crypto_devarg_params devarg_prms = { 0 };
351 struct rte_cryptodev_pmd_init_params init_params = {
352 .name = "",
353 .private_data_size = sizeof(struct mlx5_crypto_priv),
354 .socket_id = cdev->dev->numa_node,
355 .max_nb_queue_pairs =
356 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
357 };
358 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
359 int ret;
360 bool wrapped_mode;
361
362 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
363 DRV_LOG(ERR, "Non-primary process type is not supported.");
364 rte_errno = ENOTSUP;
365 return -rte_errno;
366 }
367 if (!cdev->config.hca_attr.crypto ||
368 (!cdev->config.hca_attr.aes_xts &&
369 !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) {
370 DRV_LOG(ERR, "Not enough capabilities to support crypto "
371 "operations, maybe old FW/OFED version?");
372 rte_errno = ENOTSUP;
373 return -ENOTSUP;
374 }
375 wrapped_mode = !!cdev->config.hca_attr.crypto_wrapped_import_method;
376 ret = mlx5_crypto_parse_devargs(mkvlist, &devarg_prms, wrapped_mode);
377 if (ret) {
378 DRV_LOG(ERR, "Failed to parse devargs.");
379 return -rte_errno;
380 }
381 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
382 &init_params);
383 if (crypto_dev == NULL) {
384 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
385 return -ENODEV;
386 }
387 DRV_LOG(INFO,
388 "Crypto device %s was created successfully.", ibdev_name);
389 crypto_dev->dev_ops = &mlx5_crypto_ops;
390 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode);
391 crypto_dev->driver_id = mlx5_crypto_driver_id;
392 priv = crypto_dev->data->dev_private;
393 priv->cdev = cdev;
394 priv->crypto_dev = crypto_dev;
395 priv->is_wrapped_mode = wrapped_mode;
396 priv->max_segs_num = devarg_prms.max_segs_num;
397 priv->crypto_mode = devarg_prms.crypto_mode;
398 /* Init and override AES-GCM configuration. */
399 if (devarg_prms.is_aes_gcm) {
400 ret = mlx5_crypto_gcm_init(priv);
401 if (ret) {
402 rte_cryptodev_pmd_destroy(priv->crypto_dev);
403 DRV_LOG(ERR, "Failed to init AES-GCM crypto.");
404 return -ENOTSUP;
405 }
406 } else {
407 ret = mlx5_crypto_xts_init(priv);
408 if (ret) {
409 rte_cryptodev_pmd_destroy(priv->crypto_dev);
410 DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
411 return -ENOTSUP;
412 }
413 }
414 if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
415 rte_cryptodev_pmd_destroy(priv->crypto_dev);
416 return -1;
417 }
418 if (wrapped_mode) {
419 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
420 &devarg_prms.login_attr);
421 if (login == NULL) {
422 DRV_LOG(ERR, "Failed to configure login.");
423 mlx5_devx_uar_release(&priv->uar);
424 rte_cryptodev_pmd_destroy(priv->crypto_dev);
425 return -rte_errno;
426 }
427 priv->login_obj = login;
428 }
429 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
430 DRV_LOG(INFO, "Max number of segments: %u.",
431 (unsigned int)RTE_MIN(
432 MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
433 (uint16_t)(priv->max_rdmar_ds - 2)));
434 pthread_mutex_lock(&priv_list_lock);
435 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
436 pthread_mutex_unlock(&priv_list_lock);
437
438 rte_cryptodev_pmd_probing_finish(crypto_dev);
439
440 return 0;
441 }
442
443 static int
mlx5_crypto_dev_remove(struct mlx5_common_device * cdev)444 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
445 {
446 struct mlx5_crypto_priv *priv = NULL;
447
448 pthread_mutex_lock(&priv_list_lock);
449 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
450 if (priv->crypto_dev->device == cdev->dev)
451 break;
452 if (priv)
453 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
454 pthread_mutex_unlock(&priv_list_lock);
455 if (priv) {
456 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
457 mlx5_devx_uar_release(&priv->uar);
458 rte_cryptodev_pmd_destroy(priv->crypto_dev);
459 }
460 return 0;
461 }
462
463 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
464 {
465 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
466 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
467 },
468 {
469 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
470 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
471 },
472 {
473 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
474 PCI_DEVICE_ID_MELLANOX_BLUEFIELD2)
475 },
476 {
477 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
478 PCI_DEVICE_ID_MELLANOX_CONNECTX7)
479 },
480 {
481 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
482 PCI_DEVICE_ID_MELLANOX_BLUEFIELD3)
483 },
484 {
485 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
486 PCI_DEVICE_ID_MELLANOX_CONNECTXVF)
487 },
488 {
489 .vendor_id = 0
490 }
491 };
492
493 static struct mlx5_class_driver mlx5_crypto_driver = {
494 .drv_class = MLX5_CLASS_CRYPTO,
495 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
496 .id_table = mlx5_crypto_pci_id_map,
497 .probe = mlx5_crypto_dev_probe,
498 .remove = mlx5_crypto_dev_remove,
499 };
500
RTE_INIT(rte_mlx5_crypto_init)501 RTE_INIT(rte_mlx5_crypto_init)
502 {
503 pthread_mutex_init(&priv_list_lock, NULL);
504 mlx5_common_init();
505 if (mlx5_glue != NULL)
506 mlx5_class_driver_register(&mlx5_crypto_driver);
507 }
508
509 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
510 mlx5_crypto_driver_id);
511
512 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
513 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
514 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
515 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
516