1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4 #include <ctype.h>
5 #include <stdlib.h>
6
7 #include <rte_common.h>
8 #include <rte_hexdump.h>
9 #include <rte_cryptodev.h>
10 #include <cryptodev_pmd.h>
11 #include <rte_security_driver.h>
12 #include <bus_vdev_driver.h>
13 #include <rte_malloc.h>
14 #include <rte_cpuflags.h>
15 #include <rte_reorder.h>
16 #include <rte_string_fns.h>
17
18 #include "rte_cryptodev_scheduler.h"
19 #include "scheduler_pmd_private.h"
20
21 uint8_t cryptodev_scheduler_driver_id;
22
23 struct scheduler_init_params {
24 struct rte_cryptodev_pmd_init_params def_p;
25 uint32_t nb_workers;
26 enum rte_cryptodev_scheduler_mode mode;
27 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
28 uint32_t enable_ordering;
29 uint16_t wc_pool[RTE_MAX_LCORE];
30 uint16_t nb_wc;
31 char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS]
32 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
33 };
34
35 #define RTE_CRYPTODEV_VDEV_NAME ("name")
36 #define RTE_CRYPTODEV_VDEV_WORKER ("worker")
37 #define RTE_CRYPTODEV_VDEV_MODE ("mode")
38 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param")
39 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering")
40 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs")
41 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id")
42 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask")
43 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist")
44
45 static const char * const scheduler_valid_params[] = {
46 RTE_CRYPTODEV_VDEV_NAME,
47 RTE_CRYPTODEV_VDEV_WORKER,
48 RTE_CRYPTODEV_VDEV_MODE,
49 RTE_CRYPTODEV_VDEV_MODE_PARAM,
50 RTE_CRYPTODEV_VDEV_ORDERING,
51 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
52 RTE_CRYPTODEV_VDEV_SOCKET_ID,
53 RTE_CRYPTODEV_VDEV_COREMASK,
54 RTE_CRYPTODEV_VDEV_CORELIST,
55 NULL
56 };
57
58 struct scheduler_parse_map {
59 const char *name;
60 uint32_t val;
61 };
62
63 const struct scheduler_parse_map scheduler_mode_map[] = {
64 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
65 CDEV_SCHED_MODE_ROUNDROBIN},
66 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
67 CDEV_SCHED_MODE_PKT_SIZE_DISTR},
68 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
69 CDEV_SCHED_MODE_FAILOVER},
70 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
71 CDEV_SCHED_MODE_MULTICORE}
72 };
73
74 const struct scheduler_parse_map scheduler_ordering_map[] = {
75 {"enable", 1},
76 {"disable", 0}
77 };
78
79 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':'
80
81 static void
free_mem(struct rte_cryptodev * dev)82 free_mem(struct rte_cryptodev *dev)
83 {
84 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
85 int i;
86
87 for (i = 0; i < sched_ctx->nb_init_workers; i++) {
88 rte_free(sched_ctx->init_worker_names[i]);
89 sched_ctx->init_worker_names[i] = NULL;
90 }
91
92 scheduler_free_capabilities(sched_ctx);
93
94 rte_free(dev->security_ctx);
95 dev->security_ctx = NULL;
96 }
97
98 static int
cryptodev_scheduler_create(const char * name,struct rte_vdev_device * vdev,struct scheduler_init_params * init_params)99 cryptodev_scheduler_create(const char *name,
100 struct rte_vdev_device *vdev,
101 struct scheduler_init_params *init_params)
102 {
103 struct rte_cryptodev *dev;
104 struct scheduler_ctx *sched_ctx;
105 uint32_t i;
106 int ret;
107
108 dev = rte_cryptodev_pmd_create(name, &vdev->device,
109 &init_params->def_p);
110 if (dev == NULL) {
111 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
112 name);
113 return -EFAULT;
114 }
115
116 dev->driver_id = cryptodev_scheduler_driver_id;
117 dev->dev_ops = rte_crypto_scheduler_pmd_ops;
118
119 sched_ctx = dev->data->dev_private;
120 sched_ctx->max_nb_queue_pairs =
121 init_params->def_p.max_nb_queue_pairs;
122
123 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
124 uint16_t i;
125
126 sched_ctx->nb_wc = init_params->nb_wc;
127
128 for (i = 0; i < sched_ctx->nb_wc; i++) {
129 sched_ctx->wc_pool[i] = init_params->wc_pool[i];
130 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added",
131 i, sched_ctx->wc_pool[i]);
132 }
133 }
134
135 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
136 init_params->mode < CDEV_SCHED_MODE_COUNT) {
137 union {
138 struct rte_cryptodev_scheduler_threshold_option
139 threshold_option;
140 } option;
141 enum rte_cryptodev_schedule_option_type option_type;
142 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
143 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
144 char *s, *end;
145
146 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
147 init_params->mode);
148 if (ret < 0) {
149 rte_cryptodev_pmd_release_device(dev);
150 return ret;
151 }
152
153 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
154 if (scheduler_mode_map[i].val != sched_ctx->mode)
155 continue;
156
157 CR_SCHED_LOG(INFO, " Scheduling mode = %s",
158 scheduler_mode_map[i].name);
159 break;
160 }
161
162 if (strlen(init_params->mode_param_str) > 0) {
163 s = strchr(init_params->mode_param_str,
164 CDEV_SCHED_MODE_PARAM_SEP_CHAR);
165 if (s == NULL) {
166 CR_SCHED_LOG(ERR, "Invalid mode param");
167 return -EINVAL;
168 }
169
170 strlcpy(param_name, init_params->mode_param_str,
171 s - init_params->mode_param_str + 1);
172 s++;
173 strlcpy(param_val, s,
174 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
175
176 switch (init_params->mode) {
177 case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
178 if (strcmp(param_name,
179 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
180 != 0) {
181 CR_SCHED_LOG(ERR, "Invalid mode param");
182 return -EINVAL;
183 }
184 option_type = CDEV_SCHED_OPTION_THRESHOLD;
185
186 option.threshold_option.threshold =
187 strtoul(param_val, &end, 0);
188 break;
189 default:
190 CR_SCHED_LOG(ERR, "Invalid mode param");
191 return -EINVAL;
192 }
193
194 if (sched_ctx->ops.option_set(dev, option_type,
195 (void *)&option) < 0) {
196 CR_SCHED_LOG(ERR, "Invalid mode param");
197 return -EINVAL;
198 }
199
200 CR_SCHED_LOG(INFO, " Sched mode param (%s = %s)",
201 param_name, param_val);
202 }
203 }
204
205 sched_ctx->reordering_enabled = init_params->enable_ordering;
206
207 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
208 if (scheduler_ordering_map[i].val !=
209 sched_ctx->reordering_enabled)
210 continue;
211
212 CR_SCHED_LOG(INFO, " Packet ordering = %s",
213 scheduler_ordering_map[i].name);
214
215 break;
216 }
217
218 for (i = 0; i < init_params->nb_workers; i++) {
219 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] =
220 rte_zmalloc_socket(
221 NULL,
222 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
223 SOCKET_ID_ANY);
224
225 if (!sched_ctx->init_worker_names[
226 sched_ctx->nb_init_workers]) {
227 CR_SCHED_LOG(ERR, "Not enough memory for init worker name");
228 free_mem(dev);
229 return -ENOMEM;
230 }
231
232 strncpy(sched_ctx->init_worker_names[
233 sched_ctx->nb_init_workers],
234 init_params->worker_names[i],
235 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
236
237 sched_ctx->nb_init_workers++;
238 }
239
240 /*
241 * Initialize capabilities structure as an empty structure,
242 * in case device information is requested when no workers are attached
243 */
244 sched_ctx->capabilities = rte_zmalloc_socket(NULL,
245 sizeof(struct rte_cryptodev_capabilities),
246 0, SOCKET_ID_ANY);
247
248 if (!sched_ctx->capabilities) {
249 CR_SCHED_LOG(ERR, "Not enough memory for capability information");
250 free_mem(dev);
251 return -ENOMEM;
252 }
253
254 /* Initialize security context */
255 struct rte_security_ctx *security_instance;
256 security_instance = rte_zmalloc_socket(NULL,
257 sizeof(struct rte_security_ctx),
258 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
259 if (!security_instance) {
260 CR_SCHED_LOG(ERR, "Not enough memory for security context");
261 free_mem(dev);
262 return -ENOMEM;
263 }
264
265 security_instance->device = dev;
266 security_instance->ops = rte_crypto_scheduler_pmd_sec_ops;
267 security_instance->sess_cnt = 0;
268 dev->security_ctx = security_instance;
269
270 /*
271 * Initialize security capabilities structure as an empty structure,
272 * in case device information is requested when no workers are attached
273 */
274 sched_ctx->sec_capabilities = rte_zmalloc_socket(NULL,
275 sizeof(struct rte_security_capability),
276 0, SOCKET_ID_ANY);
277
278 if (!sched_ctx->sec_capabilities) {
279 CR_SCHED_LOG(ERR, "Not enough memory for security capability information");
280 free_mem(dev);
281 return -ENOMEM;
282 }
283
284 rte_cryptodev_pmd_probing_finish(dev);
285
286 return 0;
287 }
288
289 static int
cryptodev_scheduler_remove(struct rte_vdev_device * vdev)290 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
291 {
292 const char *name;
293 struct rte_cryptodev *dev;
294 struct scheduler_ctx *sched_ctx;
295
296 if (vdev == NULL)
297 return -EINVAL;
298
299 name = rte_vdev_device_name(vdev);
300 dev = rte_cryptodev_pmd_get_named_dev(name);
301 if (dev == NULL)
302 return -EINVAL;
303
304 sched_ctx = dev->data->dev_private;
305
306 if (sched_ctx->nb_workers) {
307 uint32_t i;
308
309 for (i = 0; i < sched_ctx->nb_workers; i++)
310 rte_cryptodev_scheduler_worker_detach(dev->data->dev_id,
311 sched_ctx->workers[i].dev_id);
312 }
313
314 rte_free(dev->security_ctx);
315 dev->security_ctx = NULL;
316
317 return rte_cryptodev_pmd_destroy(dev);
318 }
319
320 /** Parse integer from integer argument */
321 static int
parse_integer_arg(const char * key __rte_unused,const char * value,void * extra_args)322 parse_integer_arg(const char *key __rte_unused,
323 const char *value, void *extra_args)
324 {
325 int *i = (int *) extra_args;
326
327 *i = atoi(value);
328 if (*i < 0) {
329 CR_SCHED_LOG(ERR, "Argument has to be positive.");
330 return -EINVAL;
331 }
332
333 return 0;
334 }
335
336 /** Parse integer from hexadecimal integer argument */
337 static int
parse_coremask_arg(const char * key __rte_unused,const char * value,void * extra_args)338 parse_coremask_arg(const char *key __rte_unused,
339 const char *value, void *extra_args)
340 {
341 int i, j, val;
342 uint16_t idx = 0;
343 char c;
344 struct scheduler_init_params *params = extra_args;
345
346 params->nb_wc = 0;
347
348 if (value == NULL)
349 return -1;
350 /* Remove all blank characters ahead and after .
351 * Remove 0x/0X if exists.
352 */
353 while (isblank(*value))
354 value++;
355 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
356 value += 2;
357 i = strlen(value);
358 while ((i > 0) && isblank(value[i - 1]))
359 i--;
360
361 if (i == 0)
362 return -1;
363
364 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
365 c = value[i];
366 if (isxdigit(c) == 0) {
367 /* invalid characters */
368 return -1;
369 }
370 if (isdigit(c))
371 val = c - '0';
372 else if (isupper(c))
373 val = c - 'A' + 10;
374 else
375 val = c - 'a' + 10;
376
377 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
378 if ((1 << j) & val)
379 params->wc_pool[params->nb_wc++] = idx;
380 }
381 }
382
383 return 0;
384 }
385
386 /** Parse integer from list of integers argument */
387 static int
parse_corelist_arg(const char * key __rte_unused,const char * value,void * extra_args)388 parse_corelist_arg(const char *key __rte_unused,
389 const char *value, void *extra_args)
390 {
391 struct scheduler_init_params *params = extra_args;
392
393 params->nb_wc = 0;
394
395 const char *token = value;
396
397 while (isdigit(token[0])) {
398 char *rval;
399 unsigned int core = strtoul(token, &rval, 10);
400
401 if (core >= RTE_MAX_LCORE) {
402 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
403 "than %u.", core, RTE_MAX_LCORE);
404 }
405 params->wc_pool[params->nb_wc++] = (uint16_t)core;
406 token = (const char *)rval;
407 if (token[0] == '\0')
408 break;
409 token++;
410 }
411
412 return 0;
413 }
414
415 /** Parse name */
416 static int
parse_name_arg(const char * key __rte_unused,const char * value,void * extra_args)417 parse_name_arg(const char *key __rte_unused,
418 const char *value, void *extra_args)
419 {
420 struct rte_cryptodev_pmd_init_params *params = extra_args;
421
422 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
423 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
424 "%u bytes.", value,
425 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
426 return -EINVAL;
427 }
428
429 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
430
431 return 0;
432 }
433
434 /** Parse worker */
435 static int
parse_worker_arg(const char * key __rte_unused,const char * value,void * extra_args)436 parse_worker_arg(const char *key __rte_unused,
437 const char *value, void *extra_args)
438 {
439 struct scheduler_init_params *param = extra_args;
440
441 if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) {
442 CR_SCHED_LOG(ERR, "Too many workers.");
443 return -ENOMEM;
444 }
445
446 strncpy(param->worker_names[param->nb_workers++], value,
447 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
448
449 return 0;
450 }
451
452 static int
parse_mode_arg(const char * key __rte_unused,const char * value,void * extra_args)453 parse_mode_arg(const char *key __rte_unused,
454 const char *value, void *extra_args)
455 {
456 struct scheduler_init_params *param = extra_args;
457 uint32_t i;
458
459 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
460 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
461 param->mode = (enum rte_cryptodev_scheduler_mode)
462 scheduler_mode_map[i].val;
463
464 break;
465 }
466 }
467
468 if (i == RTE_DIM(scheduler_mode_map)) {
469 CR_SCHED_LOG(ERR, "Unrecognized input.");
470 return -EINVAL;
471 }
472
473 return 0;
474 }
475
476 static int
parse_mode_param_arg(const char * key __rte_unused,const char * value,void * extra_args)477 parse_mode_param_arg(const char *key __rte_unused,
478 const char *value, void *extra_args)
479 {
480 struct scheduler_init_params *param = extra_args;
481
482 strlcpy(param->mode_param_str, value,
483 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
484
485 return 0;
486 }
487
488 static int
parse_ordering_arg(const char * key __rte_unused,const char * value,void * extra_args)489 parse_ordering_arg(const char *key __rte_unused,
490 const char *value, void *extra_args)
491 {
492 struct scheduler_init_params *param = extra_args;
493 uint32_t i;
494
495 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
496 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
497 param->enable_ordering =
498 scheduler_ordering_map[i].val;
499 break;
500 }
501 }
502
503 if (i == RTE_DIM(scheduler_ordering_map)) {
504 CR_SCHED_LOG(ERR, "Unrecognized input.");
505 return -EINVAL;
506 }
507
508 return 0;
509 }
510
511 static int
scheduler_parse_init_params(struct scheduler_init_params * params,const char * input_args)512 scheduler_parse_init_params(struct scheduler_init_params *params,
513 const char *input_args)
514 {
515 struct rte_kvargs *kvlist = NULL;
516 int ret = 0;
517
518 if (params == NULL)
519 return -EINVAL;
520
521 if (input_args) {
522 kvlist = rte_kvargs_parse(input_args,
523 scheduler_valid_params);
524 if (kvlist == NULL)
525 return -1;
526
527 ret = rte_kvargs_process(kvlist,
528 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
529 &parse_integer_arg,
530 ¶ms->def_p.max_nb_queue_pairs);
531 if (ret < 0)
532 goto free_kvlist;
533
534 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
535 &parse_integer_arg,
536 ¶ms->def_p.socket_id);
537 if (ret < 0)
538 goto free_kvlist;
539
540 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
541 &parse_coremask_arg,
542 params);
543 if (ret < 0)
544 goto free_kvlist;
545
546 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
547 &parse_corelist_arg,
548 params);
549 if (ret < 0)
550 goto free_kvlist;
551
552 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
553 &parse_name_arg,
554 ¶ms->def_p);
555 if (ret < 0)
556 goto free_kvlist;
557
558 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER,
559 &parse_worker_arg, params);
560 if (ret < 0)
561 goto free_kvlist;
562
563 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
564 &parse_mode_arg, params);
565 if (ret < 0)
566 goto free_kvlist;
567
568 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
569 &parse_mode_param_arg, params);
570 if (ret < 0)
571 goto free_kvlist;
572
573 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
574 &parse_ordering_arg, params);
575 if (ret < 0)
576 goto free_kvlist;
577 }
578
579 free_kvlist:
580 rte_kvargs_free(kvlist);
581 return ret;
582 }
583
584 static int
cryptodev_scheduler_probe(struct rte_vdev_device * vdev)585 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
586 {
587 struct scheduler_init_params init_params = {
588 .def_p = {
589 "",
590 sizeof(struct scheduler_ctx),
591 rte_socket_id(),
592 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
593 },
594 .nb_workers = 0,
595 .mode = CDEV_SCHED_MODE_NOT_SET,
596 .enable_ordering = 0,
597 .worker_names = { {0} }
598 };
599 const char *name;
600
601 name = rte_vdev_device_name(vdev);
602 if (name == NULL)
603 return -EINVAL;
604
605 scheduler_parse_init_params(&init_params,
606 rte_vdev_device_args(vdev));
607
608
609 return cryptodev_scheduler_create(name,
610 vdev,
611 &init_params);
612 }
613
614 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
615 .probe = cryptodev_scheduler_probe,
616 .remove = cryptodev_scheduler_remove
617 };
618
619 static struct cryptodev_driver scheduler_crypto_drv;
620
621 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
622 cryptodev_scheduler_pmd_drv);
623 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
624 "max_nb_queue_pairs=<int> "
625 "socket_id=<int> "
626 "worker=<name>");
627 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
628 cryptodev_scheduler_pmd_drv.driver,
629 cryptodev_scheduler_driver_id);
630