1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_common.h> 5 #include <rte_hexdump.h> 6 #include <rte_cryptodev.h> 7 #include <cryptodev_pmd.h> 8 #include <rte_bus_vdev.h> 9 #include <rte_malloc.h> 10 #include <rte_cpuflags.h> 11 #include <rte_reorder.h> 12 #include <rte_string_fns.h> 13 14 #include "rte_cryptodev_scheduler.h" 15 #include "scheduler_pmd_private.h" 16 17 uint8_t cryptodev_scheduler_driver_id; 18 19 struct scheduler_init_params { 20 struct rte_cryptodev_pmd_init_params def_p; 21 uint32_t nb_workers; 22 enum rte_cryptodev_scheduler_mode mode; 23 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 24 uint32_t enable_ordering; 25 uint16_t wc_pool[RTE_MAX_LCORE]; 26 uint16_t nb_wc; 27 char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS] 28 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 29 }; 30 31 #define RTE_CRYPTODEV_VDEV_NAME ("name") 32 #define RTE_CRYPTODEV_VDEV_WORKER ("worker") 33 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param") 35 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 38 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 39 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 40 41 static const char * const scheduler_valid_params[] = { 42 RTE_CRYPTODEV_VDEV_NAME, 43 RTE_CRYPTODEV_VDEV_WORKER, 44 RTE_CRYPTODEV_VDEV_MODE, 45 RTE_CRYPTODEV_VDEV_MODE_PARAM, 46 RTE_CRYPTODEV_VDEV_ORDERING, 47 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 48 RTE_CRYPTODEV_VDEV_SOCKET_ID, 49 RTE_CRYPTODEV_VDEV_COREMASK, 50 RTE_CRYPTODEV_VDEV_CORELIST 51 }; 52 53 struct scheduler_parse_map { 54 const char *name; 55 uint32_t val; 56 }; 57 58 const struct scheduler_parse_map scheduler_mode_map[] = { 59 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 60 CDEV_SCHED_MODE_ROUNDROBIN}, 61 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 62 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 63 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 64 CDEV_SCHED_MODE_FAILOVER}, 65 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 66 CDEV_SCHED_MODE_MULTICORE} 67 }; 68 69 const struct scheduler_parse_map scheduler_ordering_map[] = { 70 {"enable", 1}, 71 {"disable", 0} 72 }; 73 74 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':' 75 76 static int 77 cryptodev_scheduler_create(const char *name, 78 struct rte_vdev_device *vdev, 79 struct scheduler_init_params *init_params) 80 { 81 struct rte_cryptodev *dev; 82 struct scheduler_ctx *sched_ctx; 83 uint32_t i; 84 int ret; 85 86 dev = rte_cryptodev_pmd_create(name, &vdev->device, 87 &init_params->def_p); 88 if (dev == NULL) { 89 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 90 name); 91 return -EFAULT; 92 } 93 94 dev->driver_id = cryptodev_scheduler_driver_id; 95 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 96 97 sched_ctx = dev->data->dev_private; 98 sched_ctx->max_nb_queue_pairs = 99 init_params->def_p.max_nb_queue_pairs; 100 101 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 102 uint16_t i; 103 104 sched_ctx->nb_wc = init_params->nb_wc; 105 106 for (i = 0; i < sched_ctx->nb_wc; i++) { 107 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 108 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 109 i, sched_ctx->wc_pool[i]); 110 } 111 } 112 113 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 114 init_params->mode < CDEV_SCHED_MODE_COUNT) { 115 union { 116 struct rte_cryptodev_scheduler_threshold_option 117 threshold_option; 118 } option; 119 enum rte_cryptodev_schedule_option_type option_type; 120 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 121 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 122 char *s, *end; 123 124 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 125 init_params->mode); 126 if (ret < 0) { 127 rte_cryptodev_pmd_release_device(dev); 128 return ret; 129 } 130 131 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 132 if (scheduler_mode_map[i].val != sched_ctx->mode) 133 continue; 134 135 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 136 scheduler_mode_map[i].name); 137 break; 138 } 139 140 if (strlen(init_params->mode_param_str) > 0) { 141 s = strchr(init_params->mode_param_str, 142 CDEV_SCHED_MODE_PARAM_SEP_CHAR); 143 if (s == NULL) { 144 CR_SCHED_LOG(ERR, "Invalid mode param"); 145 return -EINVAL; 146 } 147 148 strlcpy(param_name, init_params->mode_param_str, 149 s - init_params->mode_param_str + 1); 150 s++; 151 strlcpy(param_val, s, 152 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 153 154 switch (init_params->mode) { 155 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 156 if (strcmp(param_name, 157 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES) 158 != 0) { 159 CR_SCHED_LOG(ERR, "Invalid mode param"); 160 return -EINVAL; 161 } 162 option_type = CDEV_SCHED_OPTION_THRESHOLD; 163 164 option.threshold_option.threshold = 165 strtoul(param_val, &end, 0); 166 break; 167 default: 168 CR_SCHED_LOG(ERR, "Invalid mode param"); 169 return -EINVAL; 170 } 171 172 if (sched_ctx->ops.option_set(dev, option_type, 173 (void *)&option) < 0) { 174 CR_SCHED_LOG(ERR, "Invalid mode param"); 175 return -EINVAL; 176 } 177 178 RTE_LOG(INFO, PMD, " Sched mode param (%s = %s)\n", 179 param_name, param_val); 180 } 181 } 182 183 sched_ctx->reordering_enabled = init_params->enable_ordering; 184 185 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 186 if (scheduler_ordering_map[i].val != 187 sched_ctx->reordering_enabled) 188 continue; 189 190 CR_SCHED_LOG(INFO, " Packet ordering = %s", 191 scheduler_ordering_map[i].name); 192 193 break; 194 } 195 196 for (i = 0; i < init_params->nb_workers; i++) { 197 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] = 198 rte_zmalloc_socket( 199 NULL, 200 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 201 SOCKET_ID_ANY); 202 203 if (!sched_ctx->init_worker_names[ 204 sched_ctx->nb_init_workers]) { 205 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 206 name); 207 return -ENOMEM; 208 } 209 210 strncpy(sched_ctx->init_worker_names[ 211 sched_ctx->nb_init_workers], 212 init_params->worker_names[i], 213 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 214 215 sched_ctx->nb_init_workers++; 216 } 217 218 /* 219 * Initialize capabilities structure as an empty structure, 220 * in case device information is requested when no workers are attached 221 */ 222 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 223 sizeof(struct rte_cryptodev_capabilities), 224 0, SOCKET_ID_ANY); 225 226 if (!sched_ctx->capabilities) { 227 CR_SCHED_LOG(ERR, "Not enough memory for capability " 228 "information"); 229 return -ENOMEM; 230 } 231 232 rte_cryptodev_pmd_probing_finish(dev); 233 234 return 0; 235 } 236 237 static int 238 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 239 { 240 const char *name; 241 struct rte_cryptodev *dev; 242 struct scheduler_ctx *sched_ctx; 243 244 if (vdev == NULL) 245 return -EINVAL; 246 247 name = rte_vdev_device_name(vdev); 248 dev = rte_cryptodev_pmd_get_named_dev(name); 249 if (dev == NULL) 250 return -EINVAL; 251 252 sched_ctx = dev->data->dev_private; 253 254 if (sched_ctx->nb_workers) { 255 uint32_t i; 256 257 for (i = 0; i < sched_ctx->nb_workers; i++) 258 rte_cryptodev_scheduler_worker_detach(dev->data->dev_id, 259 sched_ctx->workers[i].dev_id); 260 } 261 262 return rte_cryptodev_pmd_destroy(dev); 263 } 264 265 /** Parse integer from integer argument */ 266 static int 267 parse_integer_arg(const char *key __rte_unused, 268 const char *value, void *extra_args) 269 { 270 int *i = (int *) extra_args; 271 272 *i = atoi(value); 273 if (*i < 0) { 274 CR_SCHED_LOG(ERR, "Argument has to be positive."); 275 return -EINVAL; 276 } 277 278 return 0; 279 } 280 281 /** Parse integer from hexadecimal integer argument */ 282 static int 283 parse_coremask_arg(const char *key __rte_unused, 284 const char *value, void *extra_args) 285 { 286 int i, j, val; 287 uint16_t idx = 0; 288 char c; 289 struct scheduler_init_params *params = extra_args; 290 291 params->nb_wc = 0; 292 293 if (value == NULL) 294 return -1; 295 /* Remove all blank characters ahead and after . 296 * Remove 0x/0X if exists. 297 */ 298 while (isblank(*value)) 299 value++; 300 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 301 value += 2; 302 i = strlen(value); 303 while ((i > 0) && isblank(value[i - 1])) 304 i--; 305 306 if (i == 0) 307 return -1; 308 309 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 310 c = value[i]; 311 if (isxdigit(c) == 0) { 312 /* invalid characters */ 313 return -1; 314 } 315 if (isdigit(c)) 316 val = c - '0'; 317 else if (isupper(c)) 318 val = c - 'A' + 10; 319 else 320 val = c - 'a' + 10; 321 322 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 323 if ((1 << j) & val) 324 params->wc_pool[params->nb_wc++] = idx; 325 } 326 } 327 328 return 0; 329 } 330 331 /** Parse integer from list of integers argument */ 332 static int 333 parse_corelist_arg(const char *key __rte_unused, 334 const char *value, void *extra_args) 335 { 336 struct scheduler_init_params *params = extra_args; 337 338 params->nb_wc = 0; 339 340 const char *token = value; 341 342 while (isdigit(token[0])) { 343 char *rval; 344 unsigned int core = strtoul(token, &rval, 10); 345 346 if (core >= RTE_MAX_LCORE) { 347 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 348 "than %u.", core, RTE_MAX_LCORE); 349 } 350 params->wc_pool[params->nb_wc++] = (uint16_t)core; 351 token = (const char *)rval; 352 if (token[0] == '\0') 353 break; 354 token++; 355 } 356 357 return 0; 358 } 359 360 /** Parse name */ 361 static int 362 parse_name_arg(const char *key __rte_unused, 363 const char *value, void *extra_args) 364 { 365 struct rte_cryptodev_pmd_init_params *params = extra_args; 366 367 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 368 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 369 "%u bytes.", value, 370 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 371 return -EINVAL; 372 } 373 374 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 375 376 return 0; 377 } 378 379 /** Parse worker */ 380 static int 381 parse_worker_arg(const char *key __rte_unused, 382 const char *value, void *extra_args) 383 { 384 struct scheduler_init_params *param = extra_args; 385 386 if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) { 387 CR_SCHED_LOG(ERR, "Too many workers."); 388 return -ENOMEM; 389 } 390 391 strncpy(param->worker_names[param->nb_workers++], value, 392 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 393 394 return 0; 395 } 396 397 static int 398 parse_mode_arg(const char *key __rte_unused, 399 const char *value, void *extra_args) 400 { 401 struct scheduler_init_params *param = extra_args; 402 uint32_t i; 403 404 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 405 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 406 param->mode = (enum rte_cryptodev_scheduler_mode) 407 scheduler_mode_map[i].val; 408 409 break; 410 } 411 } 412 413 if (i == RTE_DIM(scheduler_mode_map)) { 414 CR_SCHED_LOG(ERR, "Unrecognized input."); 415 return -EINVAL; 416 } 417 418 return 0; 419 } 420 421 static int 422 parse_mode_param_arg(const char *key __rte_unused, 423 const char *value, void *extra_args) 424 { 425 struct scheduler_init_params *param = extra_args; 426 427 strlcpy(param->mode_param_str, value, 428 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 429 430 return 0; 431 } 432 433 static int 434 parse_ordering_arg(const char *key __rte_unused, 435 const char *value, void *extra_args) 436 { 437 struct scheduler_init_params *param = extra_args; 438 uint32_t i; 439 440 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 441 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 442 param->enable_ordering = 443 scheduler_ordering_map[i].val; 444 break; 445 } 446 } 447 448 if (i == RTE_DIM(scheduler_ordering_map)) { 449 CR_SCHED_LOG(ERR, "Unrecognized input."); 450 return -EINVAL; 451 } 452 453 return 0; 454 } 455 456 static int 457 scheduler_parse_init_params(struct scheduler_init_params *params, 458 const char *input_args) 459 { 460 struct rte_kvargs *kvlist = NULL; 461 int ret = 0; 462 463 if (params == NULL) 464 return -EINVAL; 465 466 if (input_args) { 467 kvlist = rte_kvargs_parse(input_args, 468 scheduler_valid_params); 469 if (kvlist == NULL) 470 return -1; 471 472 ret = rte_kvargs_process(kvlist, 473 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 474 &parse_integer_arg, 475 ¶ms->def_p.max_nb_queue_pairs); 476 if (ret < 0) 477 goto free_kvlist; 478 479 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 480 &parse_integer_arg, 481 ¶ms->def_p.socket_id); 482 if (ret < 0) 483 goto free_kvlist; 484 485 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 486 &parse_coremask_arg, 487 params); 488 if (ret < 0) 489 goto free_kvlist; 490 491 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 492 &parse_corelist_arg, 493 params); 494 if (ret < 0) 495 goto free_kvlist; 496 497 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 498 &parse_name_arg, 499 ¶ms->def_p); 500 if (ret < 0) 501 goto free_kvlist; 502 503 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER, 504 &parse_worker_arg, params); 505 if (ret < 0) 506 goto free_kvlist; 507 508 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 509 &parse_mode_arg, params); 510 if (ret < 0) 511 goto free_kvlist; 512 513 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM, 514 &parse_mode_param_arg, params); 515 if (ret < 0) 516 goto free_kvlist; 517 518 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 519 &parse_ordering_arg, params); 520 if (ret < 0) 521 goto free_kvlist; 522 } 523 524 free_kvlist: 525 rte_kvargs_free(kvlist); 526 return ret; 527 } 528 529 static int 530 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 531 { 532 struct scheduler_init_params init_params = { 533 .def_p = { 534 "", 535 sizeof(struct scheduler_ctx), 536 rte_socket_id(), 537 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 538 }, 539 .nb_workers = 0, 540 .mode = CDEV_SCHED_MODE_NOT_SET, 541 .enable_ordering = 0, 542 .worker_names = { {0} } 543 }; 544 const char *name; 545 546 name = rte_vdev_device_name(vdev); 547 if (name == NULL) 548 return -EINVAL; 549 550 scheduler_parse_init_params(&init_params, 551 rte_vdev_device_args(vdev)); 552 553 554 return cryptodev_scheduler_create(name, 555 vdev, 556 &init_params); 557 } 558 559 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 560 .probe = cryptodev_scheduler_probe, 561 .remove = cryptodev_scheduler_remove 562 }; 563 564 static struct cryptodev_driver scheduler_crypto_drv; 565 566 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 567 cryptodev_scheduler_pmd_drv); 568 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 569 "max_nb_queue_pairs=<int> " 570 "socket_id=<int> " 571 "worker=<name>"); 572 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 573 cryptodev_scheduler_pmd_drv.driver, 574 cryptodev_scheduler_driver_id); 575