1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 7 #include <rte_common.h> 8 #include <rte_hexdump.h> 9 #include <rte_cryptodev.h> 10 #include <cryptodev_pmd.h> 11 #include <rte_security_driver.h> 12 #include <bus_vdev_driver.h> 13 #include <rte_malloc.h> 14 #include <rte_cpuflags.h> 15 #include <rte_reorder.h> 16 #include <rte_string_fns.h> 17 18 #include "rte_cryptodev_scheduler.h" 19 #include "scheduler_pmd_private.h" 20 21 uint8_t cryptodev_scheduler_driver_id; 22 23 struct scheduler_init_params { 24 struct rte_cryptodev_pmd_init_params def_p; 25 uint32_t nb_workers; 26 enum rte_cryptodev_scheduler_mode mode; 27 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 28 uint32_t enable_ordering; 29 uint16_t wc_pool[RTE_MAX_LCORE]; 30 uint16_t nb_wc; 31 char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS] 32 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 33 }; 34 35 #define RTE_CRYPTODEV_VDEV_NAME ("name") 36 #define RTE_CRYPTODEV_VDEV_WORKER ("worker") 37 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 38 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param") 39 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 40 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 41 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 42 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 43 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 44 45 static const char * const scheduler_valid_params[] = { 46 RTE_CRYPTODEV_VDEV_NAME, 47 RTE_CRYPTODEV_VDEV_WORKER, 48 RTE_CRYPTODEV_VDEV_MODE, 49 RTE_CRYPTODEV_VDEV_MODE_PARAM, 50 RTE_CRYPTODEV_VDEV_ORDERING, 51 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 52 RTE_CRYPTODEV_VDEV_SOCKET_ID, 53 RTE_CRYPTODEV_VDEV_COREMASK, 54 RTE_CRYPTODEV_VDEV_CORELIST, 55 NULL 56 }; 57 58 struct scheduler_parse_map { 59 const char *name; 60 uint32_t val; 61 }; 62 63 const struct scheduler_parse_map scheduler_mode_map[] = { 64 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 65 CDEV_SCHED_MODE_ROUNDROBIN}, 66 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 67 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 68 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 69 CDEV_SCHED_MODE_FAILOVER}, 70 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 71 CDEV_SCHED_MODE_MULTICORE} 72 }; 73 74 const struct scheduler_parse_map scheduler_ordering_map[] = { 75 {"enable", 1}, 76 {"disable", 0} 77 }; 78 79 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':' 80 81 static void 82 free_mem(struct rte_cryptodev *dev) 83 { 84 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 85 int i; 86 87 for (i = 0; i < sched_ctx->nb_init_workers; i++) { 88 rte_free(sched_ctx->init_worker_names[i]); 89 sched_ctx->init_worker_names[i] = NULL; 90 } 91 92 scheduler_free_capabilities(sched_ctx); 93 94 rte_free(dev->security_ctx); 95 dev->security_ctx = NULL; 96 } 97 98 static int 99 cryptodev_scheduler_create(const char *name, 100 struct rte_vdev_device *vdev, 101 struct scheduler_init_params *init_params) 102 { 103 struct rte_cryptodev *dev; 104 struct scheduler_ctx *sched_ctx; 105 uint32_t i; 106 int ret; 107 108 dev = rte_cryptodev_pmd_create(name, &vdev->device, 109 &init_params->def_p); 110 if (dev == NULL) { 111 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 112 name); 113 return -EFAULT; 114 } 115 116 dev->driver_id = cryptodev_scheduler_driver_id; 117 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 118 119 sched_ctx = dev->data->dev_private; 120 sched_ctx->max_nb_queue_pairs = 121 init_params->def_p.max_nb_queue_pairs; 122 123 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 124 uint16_t i; 125 126 sched_ctx->nb_wc = init_params->nb_wc; 127 128 for (i = 0; i < sched_ctx->nb_wc; i++) { 129 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 130 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 131 i, sched_ctx->wc_pool[i]); 132 } 133 } 134 135 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 136 init_params->mode < CDEV_SCHED_MODE_COUNT) { 137 union { 138 struct rte_cryptodev_scheduler_threshold_option 139 threshold_option; 140 } option; 141 enum rte_cryptodev_schedule_option_type option_type; 142 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 143 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 144 char *s, *end; 145 146 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 147 init_params->mode); 148 if (ret < 0) { 149 rte_cryptodev_pmd_release_device(dev); 150 return ret; 151 } 152 153 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 154 if (scheduler_mode_map[i].val != sched_ctx->mode) 155 continue; 156 157 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 158 scheduler_mode_map[i].name); 159 break; 160 } 161 162 if (strlen(init_params->mode_param_str) > 0) { 163 s = strchr(init_params->mode_param_str, 164 CDEV_SCHED_MODE_PARAM_SEP_CHAR); 165 if (s == NULL) { 166 CR_SCHED_LOG(ERR, "Invalid mode param"); 167 return -EINVAL; 168 } 169 170 strlcpy(param_name, init_params->mode_param_str, 171 s - init_params->mode_param_str + 1); 172 s++; 173 strlcpy(param_val, s, 174 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 175 176 switch (init_params->mode) { 177 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 178 if (strcmp(param_name, 179 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES) 180 != 0) { 181 CR_SCHED_LOG(ERR, "Invalid mode param"); 182 return -EINVAL; 183 } 184 option_type = CDEV_SCHED_OPTION_THRESHOLD; 185 186 option.threshold_option.threshold = 187 strtoul(param_val, &end, 0); 188 break; 189 default: 190 CR_SCHED_LOG(ERR, "Invalid mode param"); 191 return -EINVAL; 192 } 193 194 if (sched_ctx->ops.option_set(dev, option_type, 195 (void *)&option) < 0) { 196 CR_SCHED_LOG(ERR, "Invalid mode param"); 197 return -EINVAL; 198 } 199 200 CR_SCHED_LOG(INFO, " Sched mode param (%s = %s)", 201 param_name, param_val); 202 } 203 } 204 205 sched_ctx->reordering_enabled = init_params->enable_ordering; 206 207 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 208 if (scheduler_ordering_map[i].val != 209 sched_ctx->reordering_enabled) 210 continue; 211 212 CR_SCHED_LOG(INFO, " Packet ordering = %s", 213 scheduler_ordering_map[i].name); 214 215 break; 216 } 217 218 for (i = 0; i < init_params->nb_workers; i++) { 219 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] = 220 rte_zmalloc_socket( 221 NULL, 222 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 223 SOCKET_ID_ANY); 224 225 if (!sched_ctx->init_worker_names[ 226 sched_ctx->nb_init_workers]) { 227 CR_SCHED_LOG(ERR, "Not enough memory for init worker name"); 228 free_mem(dev); 229 return -ENOMEM; 230 } 231 232 strncpy(sched_ctx->init_worker_names[ 233 sched_ctx->nb_init_workers], 234 init_params->worker_names[i], 235 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 236 237 sched_ctx->nb_init_workers++; 238 } 239 240 /* 241 * Initialize capabilities structure as an empty structure, 242 * in case device information is requested when no workers are attached 243 */ 244 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 245 sizeof(struct rte_cryptodev_capabilities), 246 0, SOCKET_ID_ANY); 247 248 if (!sched_ctx->capabilities) { 249 CR_SCHED_LOG(ERR, "Not enough memory for capability information"); 250 free_mem(dev); 251 return -ENOMEM; 252 } 253 254 /* Initialize security context */ 255 struct rte_security_ctx *security_instance; 256 security_instance = rte_zmalloc_socket(NULL, 257 sizeof(struct rte_security_ctx), 258 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 259 if (!security_instance) { 260 CR_SCHED_LOG(ERR, "Not enough memory for security context"); 261 free_mem(dev); 262 return -ENOMEM; 263 } 264 265 security_instance->device = dev; 266 security_instance->ops = rte_crypto_scheduler_pmd_sec_ops; 267 security_instance->sess_cnt = 0; 268 dev->security_ctx = security_instance; 269 270 /* 271 * Initialize security capabilities structure as an empty structure, 272 * in case device information is requested when no workers are attached 273 */ 274 sched_ctx->sec_capabilities = rte_zmalloc_socket(NULL, 275 sizeof(struct rte_security_capability), 276 0, SOCKET_ID_ANY); 277 278 if (!sched_ctx->sec_capabilities) { 279 CR_SCHED_LOG(ERR, "Not enough memory for security capability information"); 280 free_mem(dev); 281 return -ENOMEM; 282 } 283 284 rte_cryptodev_pmd_probing_finish(dev); 285 286 return 0; 287 } 288 289 static int 290 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 291 { 292 const char *name; 293 struct rte_cryptodev *dev; 294 struct scheduler_ctx *sched_ctx; 295 296 if (vdev == NULL) 297 return -EINVAL; 298 299 name = rte_vdev_device_name(vdev); 300 dev = rte_cryptodev_pmd_get_named_dev(name); 301 if (dev == NULL) 302 return -EINVAL; 303 304 sched_ctx = dev->data->dev_private; 305 306 if (sched_ctx->nb_workers) { 307 uint32_t i; 308 309 for (i = 0; i < sched_ctx->nb_workers; i++) 310 rte_cryptodev_scheduler_worker_detach(dev->data->dev_id, 311 sched_ctx->workers[i].dev_id); 312 } 313 314 rte_free(dev->security_ctx); 315 dev->security_ctx = NULL; 316 317 return rte_cryptodev_pmd_destroy(dev); 318 } 319 320 /** Parse integer from integer argument */ 321 static int 322 parse_integer_arg(const char *key __rte_unused, 323 const char *value, void *extra_args) 324 { 325 int *i = (int *) extra_args; 326 327 *i = atoi(value); 328 if (*i < 0) { 329 CR_SCHED_LOG(ERR, "Argument has to be positive."); 330 return -EINVAL; 331 } 332 333 return 0; 334 } 335 336 /** Parse integer from hexadecimal integer argument */ 337 static int 338 parse_coremask_arg(const char *key __rte_unused, 339 const char *value, void *extra_args) 340 { 341 int i, j, val; 342 uint16_t idx = 0; 343 char c; 344 struct scheduler_init_params *params = extra_args; 345 346 params->nb_wc = 0; 347 348 if (value == NULL) 349 return -1; 350 /* Remove all blank characters ahead and after . 351 * Remove 0x/0X if exists. 352 */ 353 while (isblank(*value)) 354 value++; 355 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 356 value += 2; 357 i = strlen(value); 358 while ((i > 0) && isblank(value[i - 1])) 359 i--; 360 361 if (i == 0) 362 return -1; 363 364 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 365 c = value[i]; 366 if (isxdigit(c) == 0) { 367 /* invalid characters */ 368 return -1; 369 } 370 if (isdigit(c)) 371 val = c - '0'; 372 else if (isupper(c)) 373 val = c - 'A' + 10; 374 else 375 val = c - 'a' + 10; 376 377 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 378 if ((1 << j) & val) 379 params->wc_pool[params->nb_wc++] = idx; 380 } 381 } 382 383 return 0; 384 } 385 386 /** Parse integer from list of integers argument */ 387 static int 388 parse_corelist_arg(const char *key __rte_unused, 389 const char *value, void *extra_args) 390 { 391 struct scheduler_init_params *params = extra_args; 392 393 params->nb_wc = 0; 394 395 const char *token = value; 396 397 while (isdigit(token[0])) { 398 char *rval; 399 unsigned int core = strtoul(token, &rval, 10); 400 401 if (core >= RTE_MAX_LCORE) { 402 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 403 "than %u.", core, RTE_MAX_LCORE); 404 } 405 params->wc_pool[params->nb_wc++] = (uint16_t)core; 406 token = (const char *)rval; 407 if (token[0] == '\0') 408 break; 409 token++; 410 } 411 412 return 0; 413 } 414 415 /** Parse name */ 416 static int 417 parse_name_arg(const char *key __rte_unused, 418 const char *value, void *extra_args) 419 { 420 struct rte_cryptodev_pmd_init_params *params = extra_args; 421 422 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 423 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 424 "%u bytes.", value, 425 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 426 return -EINVAL; 427 } 428 429 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 430 431 return 0; 432 } 433 434 /** Parse worker */ 435 static int 436 parse_worker_arg(const char *key __rte_unused, 437 const char *value, void *extra_args) 438 { 439 struct scheduler_init_params *param = extra_args; 440 441 if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) { 442 CR_SCHED_LOG(ERR, "Too many workers."); 443 return -ENOMEM; 444 } 445 446 strncpy(param->worker_names[param->nb_workers++], value, 447 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 448 449 return 0; 450 } 451 452 static int 453 parse_mode_arg(const char *key __rte_unused, 454 const char *value, void *extra_args) 455 { 456 struct scheduler_init_params *param = extra_args; 457 uint32_t i; 458 459 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 460 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 461 param->mode = (enum rte_cryptodev_scheduler_mode) 462 scheduler_mode_map[i].val; 463 464 break; 465 } 466 } 467 468 if (i == RTE_DIM(scheduler_mode_map)) { 469 CR_SCHED_LOG(ERR, "Unrecognized input."); 470 return -EINVAL; 471 } 472 473 return 0; 474 } 475 476 static int 477 parse_mode_param_arg(const char *key __rte_unused, 478 const char *value, void *extra_args) 479 { 480 struct scheduler_init_params *param = extra_args; 481 482 strlcpy(param->mode_param_str, value, 483 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 484 485 return 0; 486 } 487 488 static int 489 parse_ordering_arg(const char *key __rte_unused, 490 const char *value, void *extra_args) 491 { 492 struct scheduler_init_params *param = extra_args; 493 uint32_t i; 494 495 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 496 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 497 param->enable_ordering = 498 scheduler_ordering_map[i].val; 499 break; 500 } 501 } 502 503 if (i == RTE_DIM(scheduler_ordering_map)) { 504 CR_SCHED_LOG(ERR, "Unrecognized input."); 505 return -EINVAL; 506 } 507 508 return 0; 509 } 510 511 static int 512 scheduler_parse_init_params(struct scheduler_init_params *params, 513 const char *input_args) 514 { 515 struct rte_kvargs *kvlist = NULL; 516 int ret = 0; 517 518 if (params == NULL) 519 return -EINVAL; 520 521 if (input_args) { 522 kvlist = rte_kvargs_parse(input_args, 523 scheduler_valid_params); 524 if (kvlist == NULL) 525 return -1; 526 527 ret = rte_kvargs_process(kvlist, 528 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 529 &parse_integer_arg, 530 ¶ms->def_p.max_nb_queue_pairs); 531 if (ret < 0) 532 goto free_kvlist; 533 534 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 535 &parse_integer_arg, 536 ¶ms->def_p.socket_id); 537 if (ret < 0) 538 goto free_kvlist; 539 540 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 541 &parse_coremask_arg, 542 params); 543 if (ret < 0) 544 goto free_kvlist; 545 546 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 547 &parse_corelist_arg, 548 params); 549 if (ret < 0) 550 goto free_kvlist; 551 552 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 553 &parse_name_arg, 554 ¶ms->def_p); 555 if (ret < 0) 556 goto free_kvlist; 557 558 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER, 559 &parse_worker_arg, params); 560 if (ret < 0) 561 goto free_kvlist; 562 563 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 564 &parse_mode_arg, params); 565 if (ret < 0) 566 goto free_kvlist; 567 568 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM, 569 &parse_mode_param_arg, params); 570 if (ret < 0) 571 goto free_kvlist; 572 573 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 574 &parse_ordering_arg, params); 575 if (ret < 0) 576 goto free_kvlist; 577 } 578 579 free_kvlist: 580 rte_kvargs_free(kvlist); 581 return ret; 582 } 583 584 static int 585 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 586 { 587 struct scheduler_init_params init_params = { 588 .def_p = { 589 "", 590 sizeof(struct scheduler_ctx), 591 rte_socket_id(), 592 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 593 }, 594 .nb_workers = 0, 595 .mode = CDEV_SCHED_MODE_NOT_SET, 596 .enable_ordering = 0, 597 .worker_names = { {0} } 598 }; 599 const char *name; 600 601 name = rte_vdev_device_name(vdev); 602 if (name == NULL) 603 return -EINVAL; 604 605 scheduler_parse_init_params(&init_params, 606 rte_vdev_device_args(vdev)); 607 608 609 return cryptodev_scheduler_create(name, 610 vdev, 611 &init_params); 612 } 613 614 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 615 .probe = cryptodev_scheduler_probe, 616 .remove = cryptodev_scheduler_remove 617 }; 618 619 static struct cryptodev_driver scheduler_crypto_drv; 620 621 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 622 cryptodev_scheduler_pmd_drv); 623 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 624 "max_nb_queue_pairs=<int> " 625 "socket_id=<int> " 626 "worker=<name>"); 627 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 628 cryptodev_scheduler_pmd_drv.driver, 629 cryptodev_scheduler_driver_id); 630