1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 7 #include <rte_common.h> 8 #include <rte_hexdump.h> 9 #include <rte_cryptodev.h> 10 #include <cryptodev_pmd.h> 11 #include <bus_vdev_driver.h> 12 #include <rte_malloc.h> 13 #include <rte_cpuflags.h> 14 #include <rte_reorder.h> 15 #include <rte_string_fns.h> 16 17 #include "rte_cryptodev_scheduler.h" 18 #include "scheduler_pmd_private.h" 19 20 uint8_t cryptodev_scheduler_driver_id; 21 22 struct scheduler_init_params { 23 struct rte_cryptodev_pmd_init_params def_p; 24 uint32_t nb_workers; 25 enum rte_cryptodev_scheduler_mode mode; 26 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 27 uint32_t enable_ordering; 28 uint16_t wc_pool[RTE_MAX_LCORE]; 29 uint16_t nb_wc; 30 char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS] 31 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 32 }; 33 34 #define RTE_CRYPTODEV_VDEV_NAME ("name") 35 #define RTE_CRYPTODEV_VDEV_WORKER ("worker") 36 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 37 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param") 38 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 39 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 40 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 41 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 42 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 43 44 static const char * const scheduler_valid_params[] = { 45 RTE_CRYPTODEV_VDEV_NAME, 46 RTE_CRYPTODEV_VDEV_WORKER, 47 RTE_CRYPTODEV_VDEV_MODE, 48 RTE_CRYPTODEV_VDEV_MODE_PARAM, 49 RTE_CRYPTODEV_VDEV_ORDERING, 50 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 51 RTE_CRYPTODEV_VDEV_SOCKET_ID, 52 RTE_CRYPTODEV_VDEV_COREMASK, 53 RTE_CRYPTODEV_VDEV_CORELIST, 54 NULL 55 }; 56 57 struct scheduler_parse_map { 58 const char *name; 59 uint32_t val; 60 }; 61 62 const struct scheduler_parse_map scheduler_mode_map[] = { 63 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 64 CDEV_SCHED_MODE_ROUNDROBIN}, 65 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 66 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 67 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 68 CDEV_SCHED_MODE_FAILOVER}, 69 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 70 CDEV_SCHED_MODE_MULTICORE} 71 }; 72 73 const struct scheduler_parse_map scheduler_ordering_map[] = { 74 {"enable", 1}, 75 {"disable", 0} 76 }; 77 78 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':' 79 80 static int 81 cryptodev_scheduler_create(const char *name, 82 struct rte_vdev_device *vdev, 83 struct scheduler_init_params *init_params) 84 { 85 struct rte_cryptodev *dev; 86 struct scheduler_ctx *sched_ctx; 87 uint32_t i; 88 int ret; 89 90 dev = rte_cryptodev_pmd_create(name, &vdev->device, 91 &init_params->def_p); 92 if (dev == NULL) { 93 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 94 name); 95 return -EFAULT; 96 } 97 98 dev->driver_id = cryptodev_scheduler_driver_id; 99 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 100 101 sched_ctx = dev->data->dev_private; 102 sched_ctx->max_nb_queue_pairs = 103 init_params->def_p.max_nb_queue_pairs; 104 105 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 106 uint16_t i; 107 108 sched_ctx->nb_wc = init_params->nb_wc; 109 110 for (i = 0; i < sched_ctx->nb_wc; i++) { 111 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 112 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 113 i, sched_ctx->wc_pool[i]); 114 } 115 } 116 117 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 118 init_params->mode < CDEV_SCHED_MODE_COUNT) { 119 union { 120 struct rte_cryptodev_scheduler_threshold_option 121 threshold_option; 122 } option; 123 enum rte_cryptodev_schedule_option_type option_type; 124 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 125 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 126 char *s, *end; 127 128 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 129 init_params->mode); 130 if (ret < 0) { 131 rte_cryptodev_pmd_release_device(dev); 132 return ret; 133 } 134 135 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 136 if (scheduler_mode_map[i].val != sched_ctx->mode) 137 continue; 138 139 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 140 scheduler_mode_map[i].name); 141 break; 142 } 143 144 if (strlen(init_params->mode_param_str) > 0) { 145 s = strchr(init_params->mode_param_str, 146 CDEV_SCHED_MODE_PARAM_SEP_CHAR); 147 if (s == NULL) { 148 CR_SCHED_LOG(ERR, "Invalid mode param"); 149 return -EINVAL; 150 } 151 152 strlcpy(param_name, init_params->mode_param_str, 153 s - init_params->mode_param_str + 1); 154 s++; 155 strlcpy(param_val, s, 156 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 157 158 switch (init_params->mode) { 159 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 160 if (strcmp(param_name, 161 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES) 162 != 0) { 163 CR_SCHED_LOG(ERR, "Invalid mode param"); 164 return -EINVAL; 165 } 166 option_type = CDEV_SCHED_OPTION_THRESHOLD; 167 168 option.threshold_option.threshold = 169 strtoul(param_val, &end, 0); 170 break; 171 default: 172 CR_SCHED_LOG(ERR, "Invalid mode param"); 173 return -EINVAL; 174 } 175 176 if (sched_ctx->ops.option_set(dev, option_type, 177 (void *)&option) < 0) { 178 CR_SCHED_LOG(ERR, "Invalid mode param"); 179 return -EINVAL; 180 } 181 182 RTE_LOG(INFO, PMD, " Sched mode param (%s = %s)\n", 183 param_name, param_val); 184 } 185 } 186 187 sched_ctx->reordering_enabled = init_params->enable_ordering; 188 189 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 190 if (scheduler_ordering_map[i].val != 191 sched_ctx->reordering_enabled) 192 continue; 193 194 CR_SCHED_LOG(INFO, " Packet ordering = %s", 195 scheduler_ordering_map[i].name); 196 197 break; 198 } 199 200 for (i = 0; i < init_params->nb_workers; i++) { 201 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] = 202 rte_zmalloc_socket( 203 NULL, 204 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 205 SOCKET_ID_ANY); 206 207 if (!sched_ctx->init_worker_names[ 208 sched_ctx->nb_init_workers]) { 209 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 210 name); 211 return -ENOMEM; 212 } 213 214 strncpy(sched_ctx->init_worker_names[ 215 sched_ctx->nb_init_workers], 216 init_params->worker_names[i], 217 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 218 219 sched_ctx->nb_init_workers++; 220 } 221 222 /* 223 * Initialize capabilities structure as an empty structure, 224 * in case device information is requested when no workers are attached 225 */ 226 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 227 sizeof(struct rte_cryptodev_capabilities), 228 0, SOCKET_ID_ANY); 229 230 if (!sched_ctx->capabilities) { 231 CR_SCHED_LOG(ERR, "Not enough memory for capability " 232 "information"); 233 return -ENOMEM; 234 } 235 236 rte_cryptodev_pmd_probing_finish(dev); 237 238 return 0; 239 } 240 241 static int 242 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 243 { 244 const char *name; 245 struct rte_cryptodev *dev; 246 struct scheduler_ctx *sched_ctx; 247 248 if (vdev == NULL) 249 return -EINVAL; 250 251 name = rte_vdev_device_name(vdev); 252 dev = rte_cryptodev_pmd_get_named_dev(name); 253 if (dev == NULL) 254 return -EINVAL; 255 256 sched_ctx = dev->data->dev_private; 257 258 if (sched_ctx->nb_workers) { 259 uint32_t i; 260 261 for (i = 0; i < sched_ctx->nb_workers; i++) 262 rte_cryptodev_scheduler_worker_detach(dev->data->dev_id, 263 sched_ctx->workers[i].dev_id); 264 } 265 266 return rte_cryptodev_pmd_destroy(dev); 267 } 268 269 /** Parse integer from integer argument */ 270 static int 271 parse_integer_arg(const char *key __rte_unused, 272 const char *value, void *extra_args) 273 { 274 int *i = (int *) extra_args; 275 276 *i = atoi(value); 277 if (*i < 0) { 278 CR_SCHED_LOG(ERR, "Argument has to be positive."); 279 return -EINVAL; 280 } 281 282 return 0; 283 } 284 285 /** Parse integer from hexadecimal integer argument */ 286 static int 287 parse_coremask_arg(const char *key __rte_unused, 288 const char *value, void *extra_args) 289 { 290 int i, j, val; 291 uint16_t idx = 0; 292 char c; 293 struct scheduler_init_params *params = extra_args; 294 295 params->nb_wc = 0; 296 297 if (value == NULL) 298 return -1; 299 /* Remove all blank characters ahead and after . 300 * Remove 0x/0X if exists. 301 */ 302 while (isblank(*value)) 303 value++; 304 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 305 value += 2; 306 i = strlen(value); 307 while ((i > 0) && isblank(value[i - 1])) 308 i--; 309 310 if (i == 0) 311 return -1; 312 313 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 314 c = value[i]; 315 if (isxdigit(c) == 0) { 316 /* invalid characters */ 317 return -1; 318 } 319 if (isdigit(c)) 320 val = c - '0'; 321 else if (isupper(c)) 322 val = c - 'A' + 10; 323 else 324 val = c - 'a' + 10; 325 326 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 327 if ((1 << j) & val) 328 params->wc_pool[params->nb_wc++] = idx; 329 } 330 } 331 332 return 0; 333 } 334 335 /** Parse integer from list of integers argument */ 336 static int 337 parse_corelist_arg(const char *key __rte_unused, 338 const char *value, void *extra_args) 339 { 340 struct scheduler_init_params *params = extra_args; 341 342 params->nb_wc = 0; 343 344 const char *token = value; 345 346 while (isdigit(token[0])) { 347 char *rval; 348 unsigned int core = strtoul(token, &rval, 10); 349 350 if (core >= RTE_MAX_LCORE) { 351 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 352 "than %u.", core, RTE_MAX_LCORE); 353 } 354 params->wc_pool[params->nb_wc++] = (uint16_t)core; 355 token = (const char *)rval; 356 if (token[0] == '\0') 357 break; 358 token++; 359 } 360 361 return 0; 362 } 363 364 /** Parse name */ 365 static int 366 parse_name_arg(const char *key __rte_unused, 367 const char *value, void *extra_args) 368 { 369 struct rte_cryptodev_pmd_init_params *params = extra_args; 370 371 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 372 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 373 "%u bytes.", value, 374 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 375 return -EINVAL; 376 } 377 378 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 379 380 return 0; 381 } 382 383 /** Parse worker */ 384 static int 385 parse_worker_arg(const char *key __rte_unused, 386 const char *value, void *extra_args) 387 { 388 struct scheduler_init_params *param = extra_args; 389 390 if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) { 391 CR_SCHED_LOG(ERR, "Too many workers."); 392 return -ENOMEM; 393 } 394 395 strncpy(param->worker_names[param->nb_workers++], value, 396 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 397 398 return 0; 399 } 400 401 static int 402 parse_mode_arg(const char *key __rte_unused, 403 const char *value, void *extra_args) 404 { 405 struct scheduler_init_params *param = extra_args; 406 uint32_t i; 407 408 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 409 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 410 param->mode = (enum rte_cryptodev_scheduler_mode) 411 scheduler_mode_map[i].val; 412 413 break; 414 } 415 } 416 417 if (i == RTE_DIM(scheduler_mode_map)) { 418 CR_SCHED_LOG(ERR, "Unrecognized input."); 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static int 426 parse_mode_param_arg(const char *key __rte_unused, 427 const char *value, void *extra_args) 428 { 429 struct scheduler_init_params *param = extra_args; 430 431 strlcpy(param->mode_param_str, value, 432 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 433 434 return 0; 435 } 436 437 static int 438 parse_ordering_arg(const char *key __rte_unused, 439 const char *value, void *extra_args) 440 { 441 struct scheduler_init_params *param = extra_args; 442 uint32_t i; 443 444 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 445 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 446 param->enable_ordering = 447 scheduler_ordering_map[i].val; 448 break; 449 } 450 } 451 452 if (i == RTE_DIM(scheduler_ordering_map)) { 453 CR_SCHED_LOG(ERR, "Unrecognized input."); 454 return -EINVAL; 455 } 456 457 return 0; 458 } 459 460 static int 461 scheduler_parse_init_params(struct scheduler_init_params *params, 462 const char *input_args) 463 { 464 struct rte_kvargs *kvlist = NULL; 465 int ret = 0; 466 467 if (params == NULL) 468 return -EINVAL; 469 470 if (input_args) { 471 kvlist = rte_kvargs_parse(input_args, 472 scheduler_valid_params); 473 if (kvlist == NULL) 474 return -1; 475 476 ret = rte_kvargs_process(kvlist, 477 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 478 &parse_integer_arg, 479 ¶ms->def_p.max_nb_queue_pairs); 480 if (ret < 0) 481 goto free_kvlist; 482 483 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 484 &parse_integer_arg, 485 ¶ms->def_p.socket_id); 486 if (ret < 0) 487 goto free_kvlist; 488 489 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 490 &parse_coremask_arg, 491 params); 492 if (ret < 0) 493 goto free_kvlist; 494 495 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 496 &parse_corelist_arg, 497 params); 498 if (ret < 0) 499 goto free_kvlist; 500 501 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 502 &parse_name_arg, 503 ¶ms->def_p); 504 if (ret < 0) 505 goto free_kvlist; 506 507 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER, 508 &parse_worker_arg, params); 509 if (ret < 0) 510 goto free_kvlist; 511 512 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 513 &parse_mode_arg, params); 514 if (ret < 0) 515 goto free_kvlist; 516 517 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM, 518 &parse_mode_param_arg, params); 519 if (ret < 0) 520 goto free_kvlist; 521 522 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 523 &parse_ordering_arg, params); 524 if (ret < 0) 525 goto free_kvlist; 526 } 527 528 free_kvlist: 529 rte_kvargs_free(kvlist); 530 return ret; 531 } 532 533 static int 534 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 535 { 536 struct scheduler_init_params init_params = { 537 .def_p = { 538 "", 539 sizeof(struct scheduler_ctx), 540 rte_socket_id(), 541 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 542 }, 543 .nb_workers = 0, 544 .mode = CDEV_SCHED_MODE_NOT_SET, 545 .enable_ordering = 0, 546 .worker_names = { {0} } 547 }; 548 const char *name; 549 550 name = rte_vdev_device_name(vdev); 551 if (name == NULL) 552 return -EINVAL; 553 554 scheduler_parse_init_params(&init_params, 555 rte_vdev_device_args(vdev)); 556 557 558 return cryptodev_scheduler_create(name, 559 vdev, 560 &init_params); 561 } 562 563 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 564 .probe = cryptodev_scheduler_probe, 565 .remove = cryptodev_scheduler_remove 566 }; 567 568 static struct cryptodev_driver scheduler_crypto_drv; 569 570 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 571 cryptodev_scheduler_pmd_drv); 572 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 573 "max_nb_queue_pairs=<int> " 574 "socket_id=<int> " 575 "worker=<name>"); 576 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 577 cryptodev_scheduler_pmd_drv.driver, 578 cryptodev_scheduler_driver_id); 579