1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_common.h> 5 #include <rte_hexdump.h> 6 #include <rte_cryptodev.h> 7 #include <rte_cryptodev_pmd.h> 8 #include <rte_bus_vdev.h> 9 #include <rte_malloc.h> 10 #include <rte_cpuflags.h> 11 #include <rte_reorder.h> 12 13 #include "rte_cryptodev_scheduler.h" 14 #include "scheduler_pmd_private.h" 15 16 uint8_t cryptodev_driver_id; 17 18 struct scheduler_init_params { 19 struct rte_cryptodev_pmd_init_params def_p; 20 uint32_t nb_slaves; 21 enum rte_cryptodev_scheduler_mode mode; 22 uint32_t enable_ordering; 23 uint16_t wc_pool[RTE_MAX_LCORE]; 24 uint16_t nb_wc; 25 char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] 26 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 27 }; 28 29 #define RTE_CRYPTODEV_VDEV_NAME ("name") 30 #define RTE_CRYPTODEV_VDEV_SLAVE ("slave") 31 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 32 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 33 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 34 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 35 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 36 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 37 38 const char *scheduler_valid_params[] = { 39 RTE_CRYPTODEV_VDEV_NAME, 40 RTE_CRYPTODEV_VDEV_SLAVE, 41 RTE_CRYPTODEV_VDEV_MODE, 42 RTE_CRYPTODEV_VDEV_ORDERING, 43 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 44 RTE_CRYPTODEV_VDEV_SOCKET_ID, 45 RTE_CRYPTODEV_VDEV_COREMASK, 46 RTE_CRYPTODEV_VDEV_CORELIST 47 }; 48 49 struct scheduler_parse_map { 50 const char *name; 51 uint32_t val; 52 }; 53 54 const struct scheduler_parse_map scheduler_mode_map[] = { 55 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 56 CDEV_SCHED_MODE_ROUNDROBIN}, 57 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 58 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 59 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 60 CDEV_SCHED_MODE_FAILOVER}, 61 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 62 CDEV_SCHED_MODE_MULTICORE} 63 }; 64 65 const struct scheduler_parse_map scheduler_ordering_map[] = { 66 {"enable", 1}, 67 {"disable", 0} 68 }; 69 70 static int 71 cryptodev_scheduler_create(const char *name, 72 struct rte_vdev_device *vdev, 73 struct scheduler_init_params *init_params) 74 { 75 struct rte_cryptodev *dev; 76 struct scheduler_ctx *sched_ctx; 77 uint32_t i; 78 int ret; 79 80 dev = rte_cryptodev_pmd_create(name, &vdev->device, 81 &init_params->def_p); 82 if (dev == NULL) { 83 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 84 name); 85 return -EFAULT; 86 } 87 88 dev->driver_id = cryptodev_driver_id; 89 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 90 91 sched_ctx = dev->data->dev_private; 92 sched_ctx->max_nb_queue_pairs = 93 init_params->def_p.max_nb_queue_pairs; 94 95 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 96 uint16_t i; 97 98 sched_ctx->nb_wc = init_params->nb_wc; 99 100 for (i = 0; i < sched_ctx->nb_wc; i++) { 101 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 102 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 103 i, sched_ctx->wc_pool[i]); 104 } 105 } 106 107 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 108 init_params->mode < CDEV_SCHED_MODE_COUNT) { 109 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 110 init_params->mode); 111 if (ret < 0) { 112 rte_cryptodev_pmd_release_device(dev); 113 return ret; 114 } 115 116 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 117 if (scheduler_mode_map[i].val != sched_ctx->mode) 118 continue; 119 120 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 121 scheduler_mode_map[i].name); 122 break; 123 } 124 } 125 126 sched_ctx->reordering_enabled = init_params->enable_ordering; 127 128 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 129 if (scheduler_ordering_map[i].val != 130 sched_ctx->reordering_enabled) 131 continue; 132 133 CR_SCHED_LOG(INFO, " Packet ordering = %s", 134 scheduler_ordering_map[i].name); 135 136 break; 137 } 138 139 for (i = 0; i < init_params->nb_slaves; i++) { 140 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] = 141 rte_zmalloc_socket( 142 NULL, 143 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 144 SOCKET_ID_ANY); 145 146 if (!sched_ctx->init_slave_names[ 147 sched_ctx->nb_init_slaves]) { 148 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 149 name); 150 return -ENOMEM; 151 } 152 153 strncpy(sched_ctx->init_slave_names[ 154 sched_ctx->nb_init_slaves], 155 init_params->slave_names[i], 156 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 157 158 sched_ctx->nb_init_slaves++; 159 } 160 161 /* 162 * Initialize capabilities structure as an empty structure, 163 * in case device information is requested when no slaves are attached 164 */ 165 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 166 sizeof(struct rte_cryptodev_capabilities), 167 0, SOCKET_ID_ANY); 168 169 if (!sched_ctx->capabilities) { 170 CR_SCHED_LOG(ERR, "Not enough memory for capability " 171 "information"); 172 return -ENOMEM; 173 } 174 175 return 0; 176 } 177 178 static int 179 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 180 { 181 const char *name; 182 struct rte_cryptodev *dev; 183 struct scheduler_ctx *sched_ctx; 184 185 if (vdev == NULL) 186 return -EINVAL; 187 188 name = rte_vdev_device_name(vdev); 189 dev = rte_cryptodev_pmd_get_named_dev(name); 190 if (dev == NULL) 191 return -EINVAL; 192 193 sched_ctx = dev->data->dev_private; 194 195 if (sched_ctx->nb_slaves) { 196 uint32_t i; 197 198 for (i = 0; i < sched_ctx->nb_slaves; i++) 199 rte_cryptodev_scheduler_slave_detach(dev->data->dev_id, 200 sched_ctx->slaves[i].dev_id); 201 } 202 203 return rte_cryptodev_pmd_destroy(dev); 204 } 205 206 /** Parse integer from integer argument */ 207 static int 208 parse_integer_arg(const char *key __rte_unused, 209 const char *value, void *extra_args) 210 { 211 int *i = (int *) extra_args; 212 213 *i = atoi(value); 214 if (*i < 0) { 215 CR_SCHED_LOG(ERR, "Argument has to be positive."); 216 return -EINVAL; 217 } 218 219 return 0; 220 } 221 222 /** Parse integer from hexadecimal integer argument */ 223 static int 224 parse_coremask_arg(const char *key __rte_unused, 225 const char *value, void *extra_args) 226 { 227 int i, j, val; 228 uint16_t idx = 0; 229 char c; 230 struct scheduler_init_params *params = extra_args; 231 232 params->nb_wc = 0; 233 234 if (value == NULL) 235 return -1; 236 /* Remove all blank characters ahead and after . 237 * Remove 0x/0X if exists. 238 */ 239 while (isblank(*value)) 240 value++; 241 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 242 value += 2; 243 i = strlen(value); 244 while ((i > 0) && isblank(value[i - 1])) 245 i--; 246 247 if (i == 0) 248 return -1; 249 250 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 251 c = value[i]; 252 if (isxdigit(c) == 0) { 253 /* invalid characters */ 254 return -1; 255 } 256 if (isdigit(c)) 257 val = c - '0'; 258 else if (isupper(c)) 259 val = c - 'A' + 10; 260 else 261 val = c - 'a' + 10; 262 263 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 264 if ((1 << j) & val) 265 params->wc_pool[params->nb_wc++] = idx; 266 } 267 } 268 269 return 0; 270 } 271 272 /** Parse integer from list of integers argument */ 273 static int 274 parse_corelist_arg(const char *key __rte_unused, 275 const char *value, void *extra_args) 276 { 277 struct scheduler_init_params *params = extra_args; 278 279 params->nb_wc = 0; 280 281 const char *token = value; 282 283 while (isdigit(token[0])) { 284 char *rval; 285 unsigned int core = strtoul(token, &rval, 10); 286 287 if (core >= RTE_MAX_LCORE) { 288 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 289 "than %u.", core, RTE_MAX_LCORE); 290 } 291 params->wc_pool[params->nb_wc++] = (uint16_t)core; 292 token = (const char *)rval; 293 if (token[0] == '\0') 294 break; 295 token++; 296 } 297 298 return 0; 299 } 300 301 /** Parse name */ 302 static int 303 parse_name_arg(const char *key __rte_unused, 304 const char *value, void *extra_args) 305 { 306 struct rte_cryptodev_pmd_init_params *params = extra_args; 307 308 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 309 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 310 "%u bytes.", value, 311 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 312 return -EINVAL; 313 } 314 315 strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 316 317 return 0; 318 } 319 320 /** Parse slave */ 321 static int 322 parse_slave_arg(const char *key __rte_unused, 323 const char *value, void *extra_args) 324 { 325 struct scheduler_init_params *param = extra_args; 326 327 if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 328 CR_SCHED_LOG(ERR, "Too many slaves."); 329 return -ENOMEM; 330 } 331 332 strncpy(param->slave_names[param->nb_slaves++], value, 333 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 334 335 return 0; 336 } 337 338 static int 339 parse_mode_arg(const char *key __rte_unused, 340 const char *value, void *extra_args) 341 { 342 struct scheduler_init_params *param = extra_args; 343 uint32_t i; 344 345 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 346 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 347 param->mode = (enum rte_cryptodev_scheduler_mode) 348 scheduler_mode_map[i].val; 349 break; 350 } 351 } 352 353 if (i == RTE_DIM(scheduler_mode_map)) { 354 CR_SCHED_LOG(ERR, "Unrecognized input."); 355 return -EINVAL; 356 } 357 358 return 0; 359 } 360 361 static int 362 parse_ordering_arg(const char *key __rte_unused, 363 const char *value, void *extra_args) 364 { 365 struct scheduler_init_params *param = extra_args; 366 uint32_t i; 367 368 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 369 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 370 param->enable_ordering = 371 scheduler_ordering_map[i].val; 372 break; 373 } 374 } 375 376 if (i == RTE_DIM(scheduler_ordering_map)) { 377 CR_SCHED_LOG(ERR, "Unrecognized input."); 378 return -EINVAL; 379 } 380 381 return 0; 382 } 383 384 static int 385 scheduler_parse_init_params(struct scheduler_init_params *params, 386 const char *input_args) 387 { 388 struct rte_kvargs *kvlist = NULL; 389 int ret = 0; 390 391 if (params == NULL) 392 return -EINVAL; 393 394 if (input_args) { 395 kvlist = rte_kvargs_parse(input_args, 396 scheduler_valid_params); 397 if (kvlist == NULL) 398 return -1; 399 400 ret = rte_kvargs_process(kvlist, 401 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 402 &parse_integer_arg, 403 ¶ms->def_p.max_nb_queue_pairs); 404 if (ret < 0) 405 goto free_kvlist; 406 407 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 408 &parse_integer_arg, 409 ¶ms->def_p.socket_id); 410 if (ret < 0) 411 goto free_kvlist; 412 413 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 414 &parse_coremask_arg, 415 params); 416 if (ret < 0) 417 goto free_kvlist; 418 419 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 420 &parse_corelist_arg, 421 params); 422 if (ret < 0) 423 goto free_kvlist; 424 425 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 426 &parse_name_arg, 427 ¶ms->def_p); 428 if (ret < 0) 429 goto free_kvlist; 430 431 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE, 432 &parse_slave_arg, params); 433 if (ret < 0) 434 goto free_kvlist; 435 436 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 437 &parse_mode_arg, params); 438 if (ret < 0) 439 goto free_kvlist; 440 441 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 442 &parse_ordering_arg, params); 443 if (ret < 0) 444 goto free_kvlist; 445 } 446 447 free_kvlist: 448 rte_kvargs_free(kvlist); 449 return ret; 450 } 451 452 static int 453 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 454 { 455 struct scheduler_init_params init_params = { 456 .def_p = { 457 "", 458 sizeof(struct scheduler_ctx), 459 rte_socket_id(), 460 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 461 }, 462 .nb_slaves = 0, 463 .mode = CDEV_SCHED_MODE_NOT_SET, 464 .enable_ordering = 0, 465 .slave_names = { {0} } 466 }; 467 const char *name; 468 469 name = rte_vdev_device_name(vdev); 470 if (name == NULL) 471 return -EINVAL; 472 473 scheduler_parse_init_params(&init_params, 474 rte_vdev_device_args(vdev)); 475 476 477 return cryptodev_scheduler_create(name, 478 vdev, 479 &init_params); 480 } 481 482 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 483 .probe = cryptodev_scheduler_probe, 484 .remove = cryptodev_scheduler_remove 485 }; 486 487 static struct cryptodev_driver scheduler_crypto_drv; 488 489 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 490 cryptodev_scheduler_pmd_drv); 491 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 492 "max_nb_queue_pairs=<int> " 493 "socket_id=<int> " 494 "slave=<name>"); 495 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 496 cryptodev_scheduler_pmd_drv.driver, 497 cryptodev_driver_id); 498