1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 7 #include <rte_common.h> 8 #include <rte_hexdump.h> 9 #include <rte_cryptodev.h> 10 #include <cryptodev_pmd.h> 11 #include <rte_bus_vdev.h> 12 #include <rte_malloc.h> 13 #include <rte_cpuflags.h> 14 #include <rte_reorder.h> 15 #include <rte_string_fns.h> 16 17 #include "rte_cryptodev_scheduler.h" 18 #include "scheduler_pmd_private.h" 19 20 uint8_t cryptodev_scheduler_driver_id; 21 22 struct scheduler_init_params { 23 struct rte_cryptodev_pmd_init_params def_p; 24 uint32_t nb_workers; 25 enum rte_cryptodev_scheduler_mode mode; 26 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 27 uint32_t enable_ordering; 28 uint16_t wc_pool[RTE_MAX_LCORE]; 29 uint16_t nb_wc; 30 char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS] 31 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 32 }; 33 34 #define RTE_CRYPTODEV_VDEV_NAME ("name") 35 #define RTE_CRYPTODEV_VDEV_WORKER ("worker") 36 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 37 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param") 38 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 39 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 40 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 41 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 42 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 43 44 static const char * const scheduler_valid_params[] = { 45 RTE_CRYPTODEV_VDEV_NAME, 46 RTE_CRYPTODEV_VDEV_WORKER, 47 RTE_CRYPTODEV_VDEV_MODE, 48 RTE_CRYPTODEV_VDEV_MODE_PARAM, 49 RTE_CRYPTODEV_VDEV_ORDERING, 50 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 51 RTE_CRYPTODEV_VDEV_SOCKET_ID, 52 RTE_CRYPTODEV_VDEV_COREMASK, 53 RTE_CRYPTODEV_VDEV_CORELIST 54 }; 55 56 struct scheduler_parse_map { 57 const char *name; 58 uint32_t val; 59 }; 60 61 const struct scheduler_parse_map scheduler_mode_map[] = { 62 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 63 CDEV_SCHED_MODE_ROUNDROBIN}, 64 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 65 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 66 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 67 CDEV_SCHED_MODE_FAILOVER}, 68 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 69 CDEV_SCHED_MODE_MULTICORE} 70 }; 71 72 const struct scheduler_parse_map scheduler_ordering_map[] = { 73 {"enable", 1}, 74 {"disable", 0} 75 }; 76 77 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':' 78 79 static int 80 cryptodev_scheduler_create(const char *name, 81 struct rte_vdev_device *vdev, 82 struct scheduler_init_params *init_params) 83 { 84 struct rte_cryptodev *dev; 85 struct scheduler_ctx *sched_ctx; 86 uint32_t i; 87 int ret; 88 89 dev = rte_cryptodev_pmd_create(name, &vdev->device, 90 &init_params->def_p); 91 if (dev == NULL) { 92 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 93 name); 94 return -EFAULT; 95 } 96 97 dev->driver_id = cryptodev_scheduler_driver_id; 98 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 99 100 sched_ctx = dev->data->dev_private; 101 sched_ctx->max_nb_queue_pairs = 102 init_params->def_p.max_nb_queue_pairs; 103 104 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 105 uint16_t i; 106 107 sched_ctx->nb_wc = init_params->nb_wc; 108 109 for (i = 0; i < sched_ctx->nb_wc; i++) { 110 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 111 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 112 i, sched_ctx->wc_pool[i]); 113 } 114 } 115 116 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 117 init_params->mode < CDEV_SCHED_MODE_COUNT) { 118 union { 119 struct rte_cryptodev_scheduler_threshold_option 120 threshold_option; 121 } option; 122 enum rte_cryptodev_schedule_option_type option_type; 123 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 124 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 125 char *s, *end; 126 127 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 128 init_params->mode); 129 if (ret < 0) { 130 rte_cryptodev_pmd_release_device(dev); 131 return ret; 132 } 133 134 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 135 if (scheduler_mode_map[i].val != sched_ctx->mode) 136 continue; 137 138 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 139 scheduler_mode_map[i].name); 140 break; 141 } 142 143 if (strlen(init_params->mode_param_str) > 0) { 144 s = strchr(init_params->mode_param_str, 145 CDEV_SCHED_MODE_PARAM_SEP_CHAR); 146 if (s == NULL) { 147 CR_SCHED_LOG(ERR, "Invalid mode param"); 148 return -EINVAL; 149 } 150 151 strlcpy(param_name, init_params->mode_param_str, 152 s - init_params->mode_param_str + 1); 153 s++; 154 strlcpy(param_val, s, 155 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 156 157 switch (init_params->mode) { 158 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 159 if (strcmp(param_name, 160 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES) 161 != 0) { 162 CR_SCHED_LOG(ERR, "Invalid mode param"); 163 return -EINVAL; 164 } 165 option_type = CDEV_SCHED_OPTION_THRESHOLD; 166 167 option.threshold_option.threshold = 168 strtoul(param_val, &end, 0); 169 break; 170 default: 171 CR_SCHED_LOG(ERR, "Invalid mode param"); 172 return -EINVAL; 173 } 174 175 if (sched_ctx->ops.option_set(dev, option_type, 176 (void *)&option) < 0) { 177 CR_SCHED_LOG(ERR, "Invalid mode param"); 178 return -EINVAL; 179 } 180 181 RTE_LOG(INFO, PMD, " Sched mode param (%s = %s)\n", 182 param_name, param_val); 183 } 184 } 185 186 sched_ctx->reordering_enabled = init_params->enable_ordering; 187 188 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 189 if (scheduler_ordering_map[i].val != 190 sched_ctx->reordering_enabled) 191 continue; 192 193 CR_SCHED_LOG(INFO, " Packet ordering = %s", 194 scheduler_ordering_map[i].name); 195 196 break; 197 } 198 199 for (i = 0; i < init_params->nb_workers; i++) { 200 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] = 201 rte_zmalloc_socket( 202 NULL, 203 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 204 SOCKET_ID_ANY); 205 206 if (!sched_ctx->init_worker_names[ 207 sched_ctx->nb_init_workers]) { 208 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 209 name); 210 return -ENOMEM; 211 } 212 213 strncpy(sched_ctx->init_worker_names[ 214 sched_ctx->nb_init_workers], 215 init_params->worker_names[i], 216 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 217 218 sched_ctx->nb_init_workers++; 219 } 220 221 /* 222 * Initialize capabilities structure as an empty structure, 223 * in case device information is requested when no workers are attached 224 */ 225 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 226 sizeof(struct rte_cryptodev_capabilities), 227 0, SOCKET_ID_ANY); 228 229 if (!sched_ctx->capabilities) { 230 CR_SCHED_LOG(ERR, "Not enough memory for capability " 231 "information"); 232 return -ENOMEM; 233 } 234 235 rte_cryptodev_pmd_probing_finish(dev); 236 237 return 0; 238 } 239 240 static int 241 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 242 { 243 const char *name; 244 struct rte_cryptodev *dev; 245 struct scheduler_ctx *sched_ctx; 246 247 if (vdev == NULL) 248 return -EINVAL; 249 250 name = rte_vdev_device_name(vdev); 251 dev = rte_cryptodev_pmd_get_named_dev(name); 252 if (dev == NULL) 253 return -EINVAL; 254 255 sched_ctx = dev->data->dev_private; 256 257 if (sched_ctx->nb_workers) { 258 uint32_t i; 259 260 for (i = 0; i < sched_ctx->nb_workers; i++) 261 rte_cryptodev_scheduler_worker_detach(dev->data->dev_id, 262 sched_ctx->workers[i].dev_id); 263 } 264 265 return rte_cryptodev_pmd_destroy(dev); 266 } 267 268 /** Parse integer from integer argument */ 269 static int 270 parse_integer_arg(const char *key __rte_unused, 271 const char *value, void *extra_args) 272 { 273 int *i = (int *) extra_args; 274 275 *i = atoi(value); 276 if (*i < 0) { 277 CR_SCHED_LOG(ERR, "Argument has to be positive."); 278 return -EINVAL; 279 } 280 281 return 0; 282 } 283 284 /** Parse integer from hexadecimal integer argument */ 285 static int 286 parse_coremask_arg(const char *key __rte_unused, 287 const char *value, void *extra_args) 288 { 289 int i, j, val; 290 uint16_t idx = 0; 291 char c; 292 struct scheduler_init_params *params = extra_args; 293 294 params->nb_wc = 0; 295 296 if (value == NULL) 297 return -1; 298 /* Remove all blank characters ahead and after . 299 * Remove 0x/0X if exists. 300 */ 301 while (isblank(*value)) 302 value++; 303 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 304 value += 2; 305 i = strlen(value); 306 while ((i > 0) && isblank(value[i - 1])) 307 i--; 308 309 if (i == 0) 310 return -1; 311 312 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 313 c = value[i]; 314 if (isxdigit(c) == 0) { 315 /* invalid characters */ 316 return -1; 317 } 318 if (isdigit(c)) 319 val = c - '0'; 320 else if (isupper(c)) 321 val = c - 'A' + 10; 322 else 323 val = c - 'a' + 10; 324 325 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 326 if ((1 << j) & val) 327 params->wc_pool[params->nb_wc++] = idx; 328 } 329 } 330 331 return 0; 332 } 333 334 /** Parse integer from list of integers argument */ 335 static int 336 parse_corelist_arg(const char *key __rte_unused, 337 const char *value, void *extra_args) 338 { 339 struct scheduler_init_params *params = extra_args; 340 341 params->nb_wc = 0; 342 343 const char *token = value; 344 345 while (isdigit(token[0])) { 346 char *rval; 347 unsigned int core = strtoul(token, &rval, 10); 348 349 if (core >= RTE_MAX_LCORE) { 350 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 351 "than %u.", core, RTE_MAX_LCORE); 352 } 353 params->wc_pool[params->nb_wc++] = (uint16_t)core; 354 token = (const char *)rval; 355 if (token[0] == '\0') 356 break; 357 token++; 358 } 359 360 return 0; 361 } 362 363 /** Parse name */ 364 static int 365 parse_name_arg(const char *key __rte_unused, 366 const char *value, void *extra_args) 367 { 368 struct rte_cryptodev_pmd_init_params *params = extra_args; 369 370 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 371 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 372 "%u bytes.", value, 373 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 374 return -EINVAL; 375 } 376 377 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 378 379 return 0; 380 } 381 382 /** Parse worker */ 383 static int 384 parse_worker_arg(const char *key __rte_unused, 385 const char *value, void *extra_args) 386 { 387 struct scheduler_init_params *param = extra_args; 388 389 if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) { 390 CR_SCHED_LOG(ERR, "Too many workers."); 391 return -ENOMEM; 392 } 393 394 strncpy(param->worker_names[param->nb_workers++], value, 395 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 396 397 return 0; 398 } 399 400 static int 401 parse_mode_arg(const char *key __rte_unused, 402 const char *value, void *extra_args) 403 { 404 struct scheduler_init_params *param = extra_args; 405 uint32_t i; 406 407 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 408 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 409 param->mode = (enum rte_cryptodev_scheduler_mode) 410 scheduler_mode_map[i].val; 411 412 break; 413 } 414 } 415 416 if (i == RTE_DIM(scheduler_mode_map)) { 417 CR_SCHED_LOG(ERR, "Unrecognized input."); 418 return -EINVAL; 419 } 420 421 return 0; 422 } 423 424 static int 425 parse_mode_param_arg(const char *key __rte_unused, 426 const char *value, void *extra_args) 427 { 428 struct scheduler_init_params *param = extra_args; 429 430 strlcpy(param->mode_param_str, value, 431 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 432 433 return 0; 434 } 435 436 static int 437 parse_ordering_arg(const char *key __rte_unused, 438 const char *value, void *extra_args) 439 { 440 struct scheduler_init_params *param = extra_args; 441 uint32_t i; 442 443 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 444 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 445 param->enable_ordering = 446 scheduler_ordering_map[i].val; 447 break; 448 } 449 } 450 451 if (i == RTE_DIM(scheduler_ordering_map)) { 452 CR_SCHED_LOG(ERR, "Unrecognized input."); 453 return -EINVAL; 454 } 455 456 return 0; 457 } 458 459 static int 460 scheduler_parse_init_params(struct scheduler_init_params *params, 461 const char *input_args) 462 { 463 struct rte_kvargs *kvlist = NULL; 464 int ret = 0; 465 466 if (params == NULL) 467 return -EINVAL; 468 469 if (input_args) { 470 kvlist = rte_kvargs_parse(input_args, 471 scheduler_valid_params); 472 if (kvlist == NULL) 473 return -1; 474 475 ret = rte_kvargs_process(kvlist, 476 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 477 &parse_integer_arg, 478 ¶ms->def_p.max_nb_queue_pairs); 479 if (ret < 0) 480 goto free_kvlist; 481 482 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 483 &parse_integer_arg, 484 ¶ms->def_p.socket_id); 485 if (ret < 0) 486 goto free_kvlist; 487 488 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 489 &parse_coremask_arg, 490 params); 491 if (ret < 0) 492 goto free_kvlist; 493 494 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 495 &parse_corelist_arg, 496 params); 497 if (ret < 0) 498 goto free_kvlist; 499 500 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 501 &parse_name_arg, 502 ¶ms->def_p); 503 if (ret < 0) 504 goto free_kvlist; 505 506 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER, 507 &parse_worker_arg, params); 508 if (ret < 0) 509 goto free_kvlist; 510 511 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 512 &parse_mode_arg, params); 513 if (ret < 0) 514 goto free_kvlist; 515 516 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM, 517 &parse_mode_param_arg, params); 518 if (ret < 0) 519 goto free_kvlist; 520 521 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 522 &parse_ordering_arg, params); 523 if (ret < 0) 524 goto free_kvlist; 525 } 526 527 free_kvlist: 528 rte_kvargs_free(kvlist); 529 return ret; 530 } 531 532 static int 533 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 534 { 535 struct scheduler_init_params init_params = { 536 .def_p = { 537 "", 538 sizeof(struct scheduler_ctx), 539 rte_socket_id(), 540 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 541 }, 542 .nb_workers = 0, 543 .mode = CDEV_SCHED_MODE_NOT_SET, 544 .enable_ordering = 0, 545 .worker_names = { {0} } 546 }; 547 const char *name; 548 549 name = rte_vdev_device_name(vdev); 550 if (name == NULL) 551 return -EINVAL; 552 553 scheduler_parse_init_params(&init_params, 554 rte_vdev_device_args(vdev)); 555 556 557 return cryptodev_scheduler_create(name, 558 vdev, 559 &init_params); 560 } 561 562 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 563 .probe = cryptodev_scheduler_probe, 564 .remove = cryptodev_scheduler_remove 565 }; 566 567 static struct cryptodev_driver scheduler_crypto_drv; 568 569 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 570 cryptodev_scheduler_pmd_drv); 571 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 572 "max_nb_queue_pairs=<int> " 573 "socket_id=<int> " 574 "worker=<name>"); 575 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 576 cryptodev_scheduler_pmd_drv.driver, 577 cryptodev_scheduler_driver_id); 578