1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_common.h> 5 #include <rte_hexdump.h> 6 #include <rte_cryptodev.h> 7 #include <rte_cryptodev_pmd.h> 8 #include <rte_bus_vdev.h> 9 #include <rte_malloc.h> 10 #include <rte_cpuflags.h> 11 #include <rte_reorder.h> 12 #include <rte_string_fns.h> 13 14 #include "rte_cryptodev_scheduler.h" 15 #include "scheduler_pmd_private.h" 16 17 uint8_t cryptodev_scheduler_driver_id; 18 19 struct scheduler_init_params { 20 struct rte_cryptodev_pmd_init_params def_p; 21 uint32_t nb_slaves; 22 enum rte_cryptodev_scheduler_mode mode; 23 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 24 uint32_t enable_ordering; 25 uint16_t wc_pool[RTE_MAX_LCORE]; 26 uint16_t nb_wc; 27 char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] 28 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 29 }; 30 31 #define RTE_CRYPTODEV_VDEV_NAME ("name") 32 #define RTE_CRYPTODEV_VDEV_SLAVE ("slave") 33 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param") 35 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 38 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 39 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 40 41 static const char * const scheduler_valid_params[] = { 42 RTE_CRYPTODEV_VDEV_NAME, 43 RTE_CRYPTODEV_VDEV_SLAVE, 44 RTE_CRYPTODEV_VDEV_MODE, 45 RTE_CRYPTODEV_VDEV_MODE_PARAM, 46 RTE_CRYPTODEV_VDEV_ORDERING, 47 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 48 RTE_CRYPTODEV_VDEV_SOCKET_ID, 49 RTE_CRYPTODEV_VDEV_COREMASK, 50 RTE_CRYPTODEV_VDEV_CORELIST 51 }; 52 53 struct scheduler_parse_map { 54 const char *name; 55 uint32_t val; 56 }; 57 58 const struct scheduler_parse_map scheduler_mode_map[] = { 59 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 60 CDEV_SCHED_MODE_ROUNDROBIN}, 61 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 62 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 63 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 64 CDEV_SCHED_MODE_FAILOVER}, 65 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 66 CDEV_SCHED_MODE_MULTICORE} 67 }; 68 69 const struct scheduler_parse_map scheduler_ordering_map[] = { 70 {"enable", 1}, 71 {"disable", 0} 72 }; 73 74 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':' 75 76 static int 77 cryptodev_scheduler_create(const char *name, 78 struct rte_vdev_device *vdev, 79 struct scheduler_init_params *init_params) 80 { 81 struct rte_cryptodev *dev; 82 struct scheduler_ctx *sched_ctx; 83 uint32_t i; 84 int ret; 85 86 dev = rte_cryptodev_pmd_create(name, &vdev->device, 87 &init_params->def_p); 88 if (dev == NULL) { 89 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 90 name); 91 return -EFAULT; 92 } 93 94 dev->driver_id = cryptodev_scheduler_driver_id; 95 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 96 97 sched_ctx = dev->data->dev_private; 98 sched_ctx->max_nb_queue_pairs = 99 init_params->def_p.max_nb_queue_pairs; 100 101 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 102 uint16_t i; 103 104 sched_ctx->nb_wc = init_params->nb_wc; 105 106 for (i = 0; i < sched_ctx->nb_wc; i++) { 107 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 108 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 109 i, sched_ctx->wc_pool[i]); 110 } 111 } 112 113 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 114 init_params->mode < CDEV_SCHED_MODE_COUNT) { 115 union { 116 struct rte_cryptodev_scheduler_threshold_option 117 threshold_option; 118 } option; 119 enum rte_cryptodev_schedule_option_type option_type; 120 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 121 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0}; 122 char *s, *end; 123 124 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 125 init_params->mode); 126 if (ret < 0) { 127 rte_cryptodev_pmd_release_device(dev); 128 return ret; 129 } 130 131 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 132 if (scheduler_mode_map[i].val != sched_ctx->mode) 133 continue; 134 135 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 136 scheduler_mode_map[i].name); 137 break; 138 } 139 140 if (strlen(init_params->mode_param_str) > 0) { 141 s = strchr(init_params->mode_param_str, 142 CDEV_SCHED_MODE_PARAM_SEP_CHAR); 143 if (s == NULL) { 144 CR_SCHED_LOG(ERR, "Invalid mode param"); 145 return -EINVAL; 146 } 147 148 strlcpy(param_name, init_params->mode_param_str, 149 s - init_params->mode_param_str + 1); 150 s++; 151 strlcpy(param_val, s, 152 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 153 154 switch (init_params->mode) { 155 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 156 if (strcmp(param_name, 157 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES) 158 != 0) { 159 CR_SCHED_LOG(ERR, "Invalid mode param"); 160 return -EINVAL; 161 } 162 option_type = CDEV_SCHED_OPTION_THRESHOLD; 163 164 option.threshold_option.threshold = 165 strtoul(param_val, &end, 0); 166 break; 167 default: 168 CR_SCHED_LOG(ERR, "Invalid mode param"); 169 return -EINVAL; 170 } 171 172 if (sched_ctx->ops.option_set(dev, option_type, 173 (void *)&option) < 0) { 174 CR_SCHED_LOG(ERR, "Invalid mode param"); 175 return -EINVAL; 176 } 177 178 RTE_LOG(INFO, PMD, " Sched mode param (%s = %s)\n", 179 param_name, param_val); 180 } 181 } 182 183 sched_ctx->reordering_enabled = init_params->enable_ordering; 184 185 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 186 if (scheduler_ordering_map[i].val != 187 sched_ctx->reordering_enabled) 188 continue; 189 190 CR_SCHED_LOG(INFO, " Packet ordering = %s", 191 scheduler_ordering_map[i].name); 192 193 break; 194 } 195 196 for (i = 0; i < init_params->nb_slaves; i++) { 197 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] = 198 rte_zmalloc_socket( 199 NULL, 200 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 201 SOCKET_ID_ANY); 202 203 if (!sched_ctx->init_slave_names[ 204 sched_ctx->nb_init_slaves]) { 205 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 206 name); 207 return -ENOMEM; 208 } 209 210 strncpy(sched_ctx->init_slave_names[ 211 sched_ctx->nb_init_slaves], 212 init_params->slave_names[i], 213 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 214 215 sched_ctx->nb_init_slaves++; 216 } 217 218 /* 219 * Initialize capabilities structure as an empty structure, 220 * in case device information is requested when no slaves are attached 221 */ 222 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 223 sizeof(struct rte_cryptodev_capabilities), 224 0, SOCKET_ID_ANY); 225 226 if (!sched_ctx->capabilities) { 227 CR_SCHED_LOG(ERR, "Not enough memory for capability " 228 "information"); 229 return -ENOMEM; 230 } 231 232 return 0; 233 } 234 235 static int 236 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 237 { 238 const char *name; 239 struct rte_cryptodev *dev; 240 struct scheduler_ctx *sched_ctx; 241 242 if (vdev == NULL) 243 return -EINVAL; 244 245 name = rte_vdev_device_name(vdev); 246 dev = rte_cryptodev_pmd_get_named_dev(name); 247 if (dev == NULL) 248 return -EINVAL; 249 250 sched_ctx = dev->data->dev_private; 251 252 if (sched_ctx->nb_slaves) { 253 uint32_t i; 254 255 for (i = 0; i < sched_ctx->nb_slaves; i++) 256 rte_cryptodev_scheduler_slave_detach(dev->data->dev_id, 257 sched_ctx->slaves[i].dev_id); 258 } 259 260 return rte_cryptodev_pmd_destroy(dev); 261 } 262 263 /** Parse integer from integer argument */ 264 static int 265 parse_integer_arg(const char *key __rte_unused, 266 const char *value, void *extra_args) 267 { 268 int *i = (int *) extra_args; 269 270 *i = atoi(value); 271 if (*i < 0) { 272 CR_SCHED_LOG(ERR, "Argument has to be positive."); 273 return -EINVAL; 274 } 275 276 return 0; 277 } 278 279 /** Parse integer from hexadecimal integer argument */ 280 static int 281 parse_coremask_arg(const char *key __rte_unused, 282 const char *value, void *extra_args) 283 { 284 int i, j, val; 285 uint16_t idx = 0; 286 char c; 287 struct scheduler_init_params *params = extra_args; 288 289 params->nb_wc = 0; 290 291 if (value == NULL) 292 return -1; 293 /* Remove all blank characters ahead and after . 294 * Remove 0x/0X if exists. 295 */ 296 while (isblank(*value)) 297 value++; 298 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 299 value += 2; 300 i = strlen(value); 301 while ((i > 0) && isblank(value[i - 1])) 302 i--; 303 304 if (i == 0) 305 return -1; 306 307 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 308 c = value[i]; 309 if (isxdigit(c) == 0) { 310 /* invalid characters */ 311 return -1; 312 } 313 if (isdigit(c)) 314 val = c - '0'; 315 else if (isupper(c)) 316 val = c - 'A' + 10; 317 else 318 val = c - 'a' + 10; 319 320 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 321 if ((1 << j) & val) 322 params->wc_pool[params->nb_wc++] = idx; 323 } 324 } 325 326 return 0; 327 } 328 329 /** Parse integer from list of integers argument */ 330 static int 331 parse_corelist_arg(const char *key __rte_unused, 332 const char *value, void *extra_args) 333 { 334 struct scheduler_init_params *params = extra_args; 335 336 params->nb_wc = 0; 337 338 const char *token = value; 339 340 while (isdigit(token[0])) { 341 char *rval; 342 unsigned int core = strtoul(token, &rval, 10); 343 344 if (core >= RTE_MAX_LCORE) { 345 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 346 "than %u.", core, RTE_MAX_LCORE); 347 } 348 params->wc_pool[params->nb_wc++] = (uint16_t)core; 349 token = (const char *)rval; 350 if (token[0] == '\0') 351 break; 352 token++; 353 } 354 355 return 0; 356 } 357 358 /** Parse name */ 359 static int 360 parse_name_arg(const char *key __rte_unused, 361 const char *value, void *extra_args) 362 { 363 struct rte_cryptodev_pmd_init_params *params = extra_args; 364 365 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 366 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 367 "%u bytes.", value, 368 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 369 return -EINVAL; 370 } 371 372 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 373 374 return 0; 375 } 376 377 /** Parse slave */ 378 static int 379 parse_slave_arg(const char *key __rte_unused, 380 const char *value, void *extra_args) 381 { 382 struct scheduler_init_params *param = extra_args; 383 384 if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 385 CR_SCHED_LOG(ERR, "Too many slaves."); 386 return -ENOMEM; 387 } 388 389 strncpy(param->slave_names[param->nb_slaves++], value, 390 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 391 392 return 0; 393 } 394 395 static int 396 parse_mode_arg(const char *key __rte_unused, 397 const char *value, void *extra_args) 398 { 399 struct scheduler_init_params *param = extra_args; 400 uint32_t i; 401 402 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 403 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 404 param->mode = (enum rte_cryptodev_scheduler_mode) 405 scheduler_mode_map[i].val; 406 407 break; 408 } 409 } 410 411 if (i == RTE_DIM(scheduler_mode_map)) { 412 CR_SCHED_LOG(ERR, "Unrecognized input."); 413 return -EINVAL; 414 } 415 416 return 0; 417 } 418 419 static int 420 parse_mode_param_arg(const char *key __rte_unused, 421 const char *value, void *extra_args) 422 { 423 struct scheduler_init_params *param = extra_args; 424 425 strlcpy(param->mode_param_str, value, 426 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 427 428 return 0; 429 } 430 431 static int 432 parse_ordering_arg(const char *key __rte_unused, 433 const char *value, void *extra_args) 434 { 435 struct scheduler_init_params *param = extra_args; 436 uint32_t i; 437 438 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 439 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 440 param->enable_ordering = 441 scheduler_ordering_map[i].val; 442 break; 443 } 444 } 445 446 if (i == RTE_DIM(scheduler_ordering_map)) { 447 CR_SCHED_LOG(ERR, "Unrecognized input."); 448 return -EINVAL; 449 } 450 451 return 0; 452 } 453 454 static int 455 scheduler_parse_init_params(struct scheduler_init_params *params, 456 const char *input_args) 457 { 458 struct rte_kvargs *kvlist = NULL; 459 int ret = 0; 460 461 if (params == NULL) 462 return -EINVAL; 463 464 if (input_args) { 465 kvlist = rte_kvargs_parse(input_args, 466 scheduler_valid_params); 467 if (kvlist == NULL) 468 return -1; 469 470 ret = rte_kvargs_process(kvlist, 471 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 472 &parse_integer_arg, 473 ¶ms->def_p.max_nb_queue_pairs); 474 if (ret < 0) 475 goto free_kvlist; 476 477 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 478 &parse_integer_arg, 479 ¶ms->def_p.socket_id); 480 if (ret < 0) 481 goto free_kvlist; 482 483 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 484 &parse_coremask_arg, 485 params); 486 if (ret < 0) 487 goto free_kvlist; 488 489 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 490 &parse_corelist_arg, 491 params); 492 if (ret < 0) 493 goto free_kvlist; 494 495 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 496 &parse_name_arg, 497 ¶ms->def_p); 498 if (ret < 0) 499 goto free_kvlist; 500 501 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE, 502 &parse_slave_arg, params); 503 if (ret < 0) 504 goto free_kvlist; 505 506 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 507 &parse_mode_arg, params); 508 if (ret < 0) 509 goto free_kvlist; 510 511 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM, 512 &parse_mode_param_arg, params); 513 if (ret < 0) 514 goto free_kvlist; 515 516 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 517 &parse_ordering_arg, params); 518 if (ret < 0) 519 goto free_kvlist; 520 } 521 522 free_kvlist: 523 rte_kvargs_free(kvlist); 524 return ret; 525 } 526 527 static int 528 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 529 { 530 struct scheduler_init_params init_params = { 531 .def_p = { 532 "", 533 sizeof(struct scheduler_ctx), 534 rte_socket_id(), 535 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 536 }, 537 .nb_slaves = 0, 538 .mode = CDEV_SCHED_MODE_NOT_SET, 539 .enable_ordering = 0, 540 .slave_names = { {0} } 541 }; 542 const char *name; 543 544 name = rte_vdev_device_name(vdev); 545 if (name == NULL) 546 return -EINVAL; 547 548 scheduler_parse_init_params(&init_params, 549 rte_vdev_device_args(vdev)); 550 551 552 return cryptodev_scheduler_create(name, 553 vdev, 554 &init_params); 555 } 556 557 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 558 .probe = cryptodev_scheduler_probe, 559 .remove = cryptodev_scheduler_remove 560 }; 561 562 static struct cryptodev_driver scheduler_crypto_drv; 563 564 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 565 cryptodev_scheduler_pmd_drv); 566 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 567 "max_nb_queue_pairs=<int> " 568 "socket_id=<int> " 569 "slave=<name>"); 570 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 571 cryptodev_scheduler_pmd_drv.driver, 572 cryptodev_scheduler_driver_id); 573