1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_common.h> 5 #include <rte_hexdump.h> 6 #include <rte_cryptodev.h> 7 #include <rte_cryptodev_pmd.h> 8 #include <rte_bus_vdev.h> 9 #include <rte_malloc.h> 10 #include <rte_cpuflags.h> 11 #include <rte_reorder.h> 12 #include <rte_string_fns.h> 13 14 #include "rte_cryptodev_scheduler.h" 15 #include "scheduler_pmd_private.h" 16 17 uint8_t cryptodev_driver_id; 18 19 struct scheduler_init_params { 20 struct rte_cryptodev_pmd_init_params def_p; 21 uint32_t nb_slaves; 22 enum rte_cryptodev_scheduler_mode mode; 23 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 24 uint32_t enable_ordering; 25 uint16_t wc_pool[RTE_MAX_LCORE]; 26 uint16_t nb_wc; 27 char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] 28 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 29 }; 30 31 #define RTE_CRYPTODEV_VDEV_NAME ("name") 32 #define RTE_CRYPTODEV_VDEV_SLAVE ("slave") 33 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param") 35 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 38 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 39 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 40 41 const char *scheduler_valid_params[] = { 42 RTE_CRYPTODEV_VDEV_NAME, 43 RTE_CRYPTODEV_VDEV_SLAVE, 44 RTE_CRYPTODEV_VDEV_MODE, 45 RTE_CRYPTODEV_VDEV_MODE_PARAM, 46 RTE_CRYPTODEV_VDEV_ORDERING, 47 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 48 RTE_CRYPTODEV_VDEV_SOCKET_ID, 49 RTE_CRYPTODEV_VDEV_COREMASK, 50 RTE_CRYPTODEV_VDEV_CORELIST 51 }; 52 53 struct scheduler_parse_map { 54 const char *name; 55 uint32_t val; 56 }; 57 58 const struct scheduler_parse_map scheduler_mode_map[] = { 59 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 60 CDEV_SCHED_MODE_ROUNDROBIN}, 61 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 62 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 63 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 64 CDEV_SCHED_MODE_FAILOVER}, 65 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 66 CDEV_SCHED_MODE_MULTICORE} 67 }; 68 69 const struct scheduler_parse_map scheduler_ordering_map[] = { 70 {"enable", 1}, 71 {"disable", 0} 72 }; 73 74 static int 75 cryptodev_scheduler_create(const char *name, 76 struct rte_vdev_device *vdev, 77 struct scheduler_init_params *init_params) 78 { 79 struct rte_cryptodev *dev; 80 struct scheduler_ctx *sched_ctx; 81 uint32_t i; 82 int ret; 83 84 dev = rte_cryptodev_pmd_create(name, &vdev->device, 85 &init_params->def_p); 86 if (dev == NULL) { 87 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 88 name); 89 return -EFAULT; 90 } 91 92 dev->driver_id = cryptodev_driver_id; 93 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 94 95 sched_ctx = dev->data->dev_private; 96 sched_ctx->max_nb_queue_pairs = 97 init_params->def_p.max_nb_queue_pairs; 98 99 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 100 uint16_t i; 101 102 sched_ctx->nb_wc = init_params->nb_wc; 103 104 for (i = 0; i < sched_ctx->nb_wc; i++) { 105 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 106 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 107 i, sched_ctx->wc_pool[i]); 108 } 109 } 110 111 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 112 init_params->mode < CDEV_SCHED_MODE_COUNT) { 113 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 114 init_params->mode); 115 if (ret < 0) { 116 rte_cryptodev_pmd_release_device(dev); 117 return ret; 118 } 119 120 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 121 if (scheduler_mode_map[i].val != sched_ctx->mode) 122 continue; 123 124 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 125 scheduler_mode_map[i].name); 126 break; 127 } 128 } 129 130 sched_ctx->reordering_enabled = init_params->enable_ordering; 131 132 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 133 if (scheduler_ordering_map[i].val != 134 sched_ctx->reordering_enabled) 135 continue; 136 137 CR_SCHED_LOG(INFO, " Packet ordering = %s", 138 scheduler_ordering_map[i].name); 139 140 break; 141 } 142 143 for (i = 0; i < init_params->nb_slaves; i++) { 144 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] = 145 rte_zmalloc_socket( 146 NULL, 147 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 148 SOCKET_ID_ANY); 149 150 if (!sched_ctx->init_slave_names[ 151 sched_ctx->nb_init_slaves]) { 152 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 153 name); 154 return -ENOMEM; 155 } 156 157 strncpy(sched_ctx->init_slave_names[ 158 sched_ctx->nb_init_slaves], 159 init_params->slave_names[i], 160 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 161 162 sched_ctx->nb_init_slaves++; 163 } 164 165 /* 166 * Initialize capabilities structure as an empty structure, 167 * in case device information is requested when no slaves are attached 168 */ 169 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 170 sizeof(struct rte_cryptodev_capabilities), 171 0, SOCKET_ID_ANY); 172 173 if (!sched_ctx->capabilities) { 174 CR_SCHED_LOG(ERR, "Not enough memory for capability " 175 "information"); 176 return -ENOMEM; 177 } 178 179 return 0; 180 } 181 182 static int 183 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 184 { 185 const char *name; 186 struct rte_cryptodev *dev; 187 struct scheduler_ctx *sched_ctx; 188 189 if (vdev == NULL) 190 return -EINVAL; 191 192 name = rte_vdev_device_name(vdev); 193 dev = rte_cryptodev_pmd_get_named_dev(name); 194 if (dev == NULL) 195 return -EINVAL; 196 197 sched_ctx = dev->data->dev_private; 198 199 if (sched_ctx->nb_slaves) { 200 uint32_t i; 201 202 for (i = 0; i < sched_ctx->nb_slaves; i++) 203 rte_cryptodev_scheduler_slave_detach(dev->data->dev_id, 204 sched_ctx->slaves[i].dev_id); 205 } 206 207 return rte_cryptodev_pmd_destroy(dev); 208 } 209 210 /** Parse integer from integer argument */ 211 static int 212 parse_integer_arg(const char *key __rte_unused, 213 const char *value, void *extra_args) 214 { 215 int *i = (int *) extra_args; 216 217 *i = atoi(value); 218 if (*i < 0) { 219 CR_SCHED_LOG(ERR, "Argument has to be positive."); 220 return -EINVAL; 221 } 222 223 return 0; 224 } 225 226 /** Parse integer from hexadecimal integer argument */ 227 static int 228 parse_coremask_arg(const char *key __rte_unused, 229 const char *value, void *extra_args) 230 { 231 int i, j, val; 232 uint16_t idx = 0; 233 char c; 234 struct scheduler_init_params *params = extra_args; 235 236 params->nb_wc = 0; 237 238 if (value == NULL) 239 return -1; 240 /* Remove all blank characters ahead and after . 241 * Remove 0x/0X if exists. 242 */ 243 while (isblank(*value)) 244 value++; 245 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 246 value += 2; 247 i = strlen(value); 248 while ((i > 0) && isblank(value[i - 1])) 249 i--; 250 251 if (i == 0) 252 return -1; 253 254 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 255 c = value[i]; 256 if (isxdigit(c) == 0) { 257 /* invalid characters */ 258 return -1; 259 } 260 if (isdigit(c)) 261 val = c - '0'; 262 else if (isupper(c)) 263 val = c - 'A' + 10; 264 else 265 val = c - 'a' + 10; 266 267 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 268 if ((1 << j) & val) 269 params->wc_pool[params->nb_wc++] = idx; 270 } 271 } 272 273 return 0; 274 } 275 276 /** Parse integer from list of integers argument */ 277 static int 278 parse_corelist_arg(const char *key __rte_unused, 279 const char *value, void *extra_args) 280 { 281 struct scheduler_init_params *params = extra_args; 282 283 params->nb_wc = 0; 284 285 const char *token = value; 286 287 while (isdigit(token[0])) { 288 char *rval; 289 unsigned int core = strtoul(token, &rval, 10); 290 291 if (core >= RTE_MAX_LCORE) { 292 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 293 "than %u.", core, RTE_MAX_LCORE); 294 } 295 params->wc_pool[params->nb_wc++] = (uint16_t)core; 296 token = (const char *)rval; 297 if (token[0] == '\0') 298 break; 299 token++; 300 } 301 302 return 0; 303 } 304 305 /** Parse name */ 306 static int 307 parse_name_arg(const char *key __rte_unused, 308 const char *value, void *extra_args) 309 { 310 struct rte_cryptodev_pmd_init_params *params = extra_args; 311 312 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 313 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 314 "%u bytes.", value, 315 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 316 return -EINVAL; 317 } 318 319 strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 320 321 return 0; 322 } 323 324 /** Parse slave */ 325 static int 326 parse_slave_arg(const char *key __rte_unused, 327 const char *value, void *extra_args) 328 { 329 struct scheduler_init_params *param = extra_args; 330 331 if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 332 CR_SCHED_LOG(ERR, "Too many slaves."); 333 return -ENOMEM; 334 } 335 336 strncpy(param->slave_names[param->nb_slaves++], value, 337 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 338 339 return 0; 340 } 341 342 static int 343 parse_mode_arg(const char *key __rte_unused, 344 const char *value, void *extra_args) 345 { 346 struct scheduler_init_params *param = extra_args; 347 uint32_t i; 348 349 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 350 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 351 param->mode = (enum rte_cryptodev_scheduler_mode) 352 scheduler_mode_map[i].val; 353 354 break; 355 } 356 } 357 358 if (i == RTE_DIM(scheduler_mode_map)) { 359 CR_SCHED_LOG(ERR, "Unrecognized input."); 360 return -EINVAL; 361 } 362 363 return 0; 364 } 365 366 static int 367 parse_mode_param_arg(const char *key __rte_unused, 368 const char *value, void *extra_args) 369 { 370 struct scheduler_init_params *param = extra_args; 371 372 strlcpy(param->mode_param_str, value, 373 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 374 375 return 0; 376 } 377 378 static int 379 parse_ordering_arg(const char *key __rte_unused, 380 const char *value, void *extra_args) 381 { 382 struct scheduler_init_params *param = extra_args; 383 uint32_t i; 384 385 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 386 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 387 param->enable_ordering = 388 scheduler_ordering_map[i].val; 389 break; 390 } 391 } 392 393 if (i == RTE_DIM(scheduler_ordering_map)) { 394 CR_SCHED_LOG(ERR, "Unrecognized input."); 395 return -EINVAL; 396 } 397 398 return 0; 399 } 400 401 static int 402 scheduler_parse_init_params(struct scheduler_init_params *params, 403 const char *input_args) 404 { 405 struct rte_kvargs *kvlist = NULL; 406 int ret = 0; 407 408 if (params == NULL) 409 return -EINVAL; 410 411 if (input_args) { 412 kvlist = rte_kvargs_parse(input_args, 413 scheduler_valid_params); 414 if (kvlist == NULL) 415 return -1; 416 417 ret = rte_kvargs_process(kvlist, 418 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 419 &parse_integer_arg, 420 ¶ms->def_p.max_nb_queue_pairs); 421 if (ret < 0) 422 goto free_kvlist; 423 424 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 425 &parse_integer_arg, 426 ¶ms->def_p.socket_id); 427 if (ret < 0) 428 goto free_kvlist; 429 430 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 431 &parse_coremask_arg, 432 params); 433 if (ret < 0) 434 goto free_kvlist; 435 436 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 437 &parse_corelist_arg, 438 params); 439 if (ret < 0) 440 goto free_kvlist; 441 442 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 443 &parse_name_arg, 444 ¶ms->def_p); 445 if (ret < 0) 446 goto free_kvlist; 447 448 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE, 449 &parse_slave_arg, params); 450 if (ret < 0) 451 goto free_kvlist; 452 453 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 454 &parse_mode_arg, params); 455 if (ret < 0) 456 goto free_kvlist; 457 458 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM, 459 &parse_mode_param_arg, params); 460 if (ret < 0) 461 goto free_kvlist; 462 463 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 464 &parse_ordering_arg, params); 465 if (ret < 0) 466 goto free_kvlist; 467 } 468 469 free_kvlist: 470 rte_kvargs_free(kvlist); 471 return ret; 472 } 473 474 static int 475 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 476 { 477 struct scheduler_init_params init_params = { 478 .def_p = { 479 "", 480 sizeof(struct scheduler_ctx), 481 rte_socket_id(), 482 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 483 }, 484 .nb_slaves = 0, 485 .mode = CDEV_SCHED_MODE_NOT_SET, 486 .enable_ordering = 0, 487 .slave_names = { {0} } 488 }; 489 const char *name; 490 491 name = rte_vdev_device_name(vdev); 492 if (name == NULL) 493 return -EINVAL; 494 495 scheduler_parse_init_params(&init_params, 496 rte_vdev_device_args(vdev)); 497 498 499 return cryptodev_scheduler_create(name, 500 vdev, 501 &init_params); 502 } 503 504 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 505 .probe = cryptodev_scheduler_probe, 506 .remove = cryptodev_scheduler_remove 507 }; 508 509 static struct cryptodev_driver scheduler_crypto_drv; 510 511 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 512 cryptodev_scheduler_pmd_drv); 513 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 514 "max_nb_queue_pairs=<int> " 515 "socket_id=<int> " 516 "slave=<name>"); 517 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 518 cryptodev_scheduler_pmd_drv.driver, 519 cryptodev_driver_id); 520