1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_common.h> 5 #include <rte_hexdump.h> 6 #include <rte_cryptodev.h> 7 #include <rte_cryptodev_pmd.h> 8 #include <rte_bus_vdev.h> 9 #include <rte_malloc.h> 10 #include <rte_cpuflags.h> 11 #include <rte_reorder.h> 12 13 #include "rte_cryptodev_scheduler.h" 14 #include "scheduler_pmd_private.h" 15 16 uint8_t cryptodev_driver_id; 17 18 struct scheduler_init_params { 19 struct rte_cryptodev_pmd_init_params def_p; 20 uint32_t nb_slaves; 21 enum rte_cryptodev_scheduler_mode mode; 22 uint32_t enable_ordering; 23 uint16_t wc_pool[RTE_MAX_LCORE]; 24 uint16_t nb_wc; 25 char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] 26 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 27 }; 28 29 #define RTE_CRYPTODEV_VDEV_NAME ("name") 30 #define RTE_CRYPTODEV_VDEV_SLAVE ("slave") 31 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 32 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 33 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 34 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG ("max_nb_sessions") 35 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 36 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 37 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 38 39 const char *scheduler_valid_params[] = { 40 RTE_CRYPTODEV_VDEV_NAME, 41 RTE_CRYPTODEV_VDEV_SLAVE, 42 RTE_CRYPTODEV_VDEV_MODE, 43 RTE_CRYPTODEV_VDEV_ORDERING, 44 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 45 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, 46 RTE_CRYPTODEV_VDEV_SOCKET_ID, 47 RTE_CRYPTODEV_VDEV_COREMASK, 48 RTE_CRYPTODEV_VDEV_CORELIST 49 }; 50 51 struct scheduler_parse_map { 52 const char *name; 53 uint32_t val; 54 }; 55 56 const struct scheduler_parse_map scheduler_mode_map[] = { 57 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 58 CDEV_SCHED_MODE_ROUNDROBIN}, 59 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 60 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 61 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 62 CDEV_SCHED_MODE_FAILOVER}, 63 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 64 CDEV_SCHED_MODE_MULTICORE} 65 }; 66 67 const struct scheduler_parse_map scheduler_ordering_map[] = { 68 {"enable", 1}, 69 {"disable", 0} 70 }; 71 72 static int 73 cryptodev_scheduler_create(const char *name, 74 struct rte_vdev_device *vdev, 75 struct scheduler_init_params *init_params) 76 { 77 struct rte_cryptodev *dev; 78 struct scheduler_ctx *sched_ctx; 79 uint32_t i; 80 int ret; 81 82 dev = rte_cryptodev_pmd_create(name, &vdev->device, 83 &init_params->def_p); 84 if (dev == NULL) { 85 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev", 86 name); 87 return -EFAULT; 88 } 89 90 dev->driver_id = cryptodev_driver_id; 91 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 92 93 sched_ctx = dev->data->dev_private; 94 sched_ctx->max_nb_queue_pairs = 95 init_params->def_p.max_nb_queue_pairs; 96 97 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 98 uint16_t i; 99 100 sched_ctx->nb_wc = init_params->nb_wc; 101 102 for (i = 0; i < sched_ctx->nb_wc; i++) { 103 sched_ctx->wc_pool[i] = init_params->wc_pool[i]; 104 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added", 105 i, sched_ctx->wc_pool[i]); 106 } 107 } 108 109 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 110 init_params->mode < CDEV_SCHED_MODE_COUNT) { 111 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 112 init_params->mode); 113 if (ret < 0) { 114 rte_cryptodev_pmd_release_device(dev); 115 return ret; 116 } 117 118 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 119 if (scheduler_mode_map[i].val != sched_ctx->mode) 120 continue; 121 122 CR_SCHED_LOG(INFO, " Scheduling mode = %s", 123 scheduler_mode_map[i].name); 124 break; 125 } 126 } 127 128 sched_ctx->reordering_enabled = init_params->enable_ordering; 129 130 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 131 if (scheduler_ordering_map[i].val != 132 sched_ctx->reordering_enabled) 133 continue; 134 135 CR_SCHED_LOG(INFO, " Packet ordering = %s", 136 scheduler_ordering_map[i].name); 137 138 break; 139 } 140 141 for (i = 0; i < init_params->nb_slaves; i++) { 142 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] = 143 rte_zmalloc_socket( 144 NULL, 145 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 146 SOCKET_ID_ANY); 147 148 if (!sched_ctx->init_slave_names[ 149 sched_ctx->nb_init_slaves]) { 150 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory", 151 name); 152 return -ENOMEM; 153 } 154 155 strncpy(sched_ctx->init_slave_names[ 156 sched_ctx->nb_init_slaves], 157 init_params->slave_names[i], 158 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 159 160 sched_ctx->nb_init_slaves++; 161 } 162 163 /* 164 * Initialize capabilities structure as an empty structure, 165 * in case device information is requested when no slaves are attached 166 */ 167 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 168 sizeof(struct rte_cryptodev_capabilities), 169 0, SOCKET_ID_ANY); 170 171 if (!sched_ctx->capabilities) { 172 CR_SCHED_LOG(ERR, "Not enough memory for capability " 173 "information"); 174 return -ENOMEM; 175 } 176 177 return 0; 178 } 179 180 static int 181 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 182 { 183 const char *name; 184 struct rte_cryptodev *dev; 185 struct scheduler_ctx *sched_ctx; 186 187 if (vdev == NULL) 188 return -EINVAL; 189 190 name = rte_vdev_device_name(vdev); 191 dev = rte_cryptodev_pmd_get_named_dev(name); 192 if (dev == NULL) 193 return -EINVAL; 194 195 sched_ctx = dev->data->dev_private; 196 197 if (sched_ctx->nb_slaves) { 198 uint32_t i; 199 200 for (i = 0; i < sched_ctx->nb_slaves; i++) 201 rte_cryptodev_scheduler_slave_detach(dev->data->dev_id, 202 sched_ctx->slaves[i].dev_id); 203 } 204 205 return rte_cryptodev_pmd_destroy(dev); 206 } 207 208 /** Parse integer from integer argument */ 209 static int 210 parse_integer_arg(const char *key __rte_unused, 211 const char *value, void *extra_args) 212 { 213 int *i = (int *) extra_args; 214 215 *i = atoi(value); 216 if (*i < 0) { 217 CR_SCHED_LOG(ERR, "Argument has to be positive."); 218 return -EINVAL; 219 } 220 221 return 0; 222 } 223 224 /** Parse integer from hexadecimal integer argument */ 225 static int 226 parse_coremask_arg(const char *key __rte_unused, 227 const char *value, void *extra_args) 228 { 229 int i, j, val; 230 uint16_t idx = 0; 231 char c; 232 struct scheduler_init_params *params = extra_args; 233 234 params->nb_wc = 0; 235 236 if (value == NULL) 237 return -1; 238 /* Remove all blank characters ahead and after . 239 * Remove 0x/0X if exists. 240 */ 241 while (isblank(*value)) 242 value++; 243 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) 244 value += 2; 245 i = strlen(value); 246 while ((i > 0) && isblank(value[i - 1])) 247 i--; 248 249 if (i == 0) 250 return -1; 251 252 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { 253 c = value[i]; 254 if (isxdigit(c) == 0) { 255 /* invalid characters */ 256 return -1; 257 } 258 if (isdigit(c)) 259 val = c - '0'; 260 else if (isupper(c)) 261 val = c - 'A' + 10; 262 else 263 val = c - 'a' + 10; 264 265 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { 266 if ((1 << j) & val) 267 params->wc_pool[params->nb_wc++] = idx; 268 } 269 } 270 271 return 0; 272 } 273 274 /** Parse integer from list of integers argument */ 275 static int 276 parse_corelist_arg(const char *key __rte_unused, 277 const char *value, void *extra_args) 278 { 279 struct scheduler_init_params *params = extra_args; 280 281 params->nb_wc = 0; 282 283 const char *token = value; 284 285 while (isdigit(token[0])) { 286 char *rval; 287 unsigned int core = strtoul(token, &rval, 10); 288 289 if (core >= RTE_MAX_LCORE) { 290 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller " 291 "than %u.", core, RTE_MAX_LCORE); 292 } 293 params->wc_pool[params->nb_wc++] = (uint16_t)core; 294 token = (const char *)rval; 295 if (token[0] == '\0') 296 break; 297 token++; 298 } 299 300 return 0; 301 } 302 303 /** Parse name */ 304 static int 305 parse_name_arg(const char *key __rte_unused, 306 const char *value, void *extra_args) 307 { 308 struct rte_cryptodev_pmd_init_params *params = extra_args; 309 310 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 311 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 312 "%u bytes.", value, 313 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 314 return -EINVAL; 315 } 316 317 strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 318 319 return 0; 320 } 321 322 /** Parse slave */ 323 static int 324 parse_slave_arg(const char *key __rte_unused, 325 const char *value, void *extra_args) 326 { 327 struct scheduler_init_params *param = extra_args; 328 329 if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 330 CR_SCHED_LOG(ERR, "Too many slaves."); 331 return -ENOMEM; 332 } 333 334 strncpy(param->slave_names[param->nb_slaves++], value, 335 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 336 337 return 0; 338 } 339 340 static int 341 parse_mode_arg(const char *key __rte_unused, 342 const char *value, void *extra_args) 343 { 344 struct scheduler_init_params *param = extra_args; 345 uint32_t i; 346 347 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 348 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 349 param->mode = (enum rte_cryptodev_scheduler_mode) 350 scheduler_mode_map[i].val; 351 break; 352 } 353 } 354 355 if (i == RTE_DIM(scheduler_mode_map)) { 356 CR_SCHED_LOG(ERR, "Unrecognized input."); 357 return -EINVAL; 358 } 359 360 return 0; 361 } 362 363 static int 364 parse_ordering_arg(const char *key __rte_unused, 365 const char *value, void *extra_args) 366 { 367 struct scheduler_init_params *param = extra_args; 368 uint32_t i; 369 370 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 371 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 372 param->enable_ordering = 373 scheduler_ordering_map[i].val; 374 break; 375 } 376 } 377 378 if (i == RTE_DIM(scheduler_ordering_map)) { 379 CR_SCHED_LOG(ERR, "Unrecognized input."); 380 return -EINVAL; 381 } 382 383 return 0; 384 } 385 386 static int 387 scheduler_parse_init_params(struct scheduler_init_params *params, 388 const char *input_args) 389 { 390 struct rte_kvargs *kvlist = NULL; 391 int ret = 0; 392 393 if (params == NULL) 394 return -EINVAL; 395 396 if (input_args) { 397 kvlist = rte_kvargs_parse(input_args, 398 scheduler_valid_params); 399 if (kvlist == NULL) 400 return -1; 401 402 ret = rte_kvargs_process(kvlist, 403 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 404 &parse_integer_arg, 405 ¶ms->def_p.max_nb_queue_pairs); 406 if (ret < 0) 407 goto free_kvlist; 408 409 ret = rte_kvargs_process(kvlist, 410 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, 411 &parse_integer_arg, 412 ¶ms->def_p.max_nb_sessions); 413 if (ret < 0) 414 goto free_kvlist; 415 416 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 417 &parse_integer_arg, 418 ¶ms->def_p.socket_id); 419 if (ret < 0) 420 goto free_kvlist; 421 422 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 423 &parse_coremask_arg, 424 params); 425 if (ret < 0) 426 goto free_kvlist; 427 428 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 429 &parse_corelist_arg, 430 params); 431 if (ret < 0) 432 goto free_kvlist; 433 434 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 435 &parse_name_arg, 436 ¶ms->def_p); 437 if (ret < 0) 438 goto free_kvlist; 439 440 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE, 441 &parse_slave_arg, params); 442 if (ret < 0) 443 goto free_kvlist; 444 445 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 446 &parse_mode_arg, params); 447 if (ret < 0) 448 goto free_kvlist; 449 450 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 451 &parse_ordering_arg, params); 452 if (ret < 0) 453 goto free_kvlist; 454 } 455 456 free_kvlist: 457 rte_kvargs_free(kvlist); 458 return ret; 459 } 460 461 static int 462 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 463 { 464 struct scheduler_init_params init_params = { 465 .def_p = { 466 "", 467 sizeof(struct scheduler_ctx), 468 rte_socket_id(), 469 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 470 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS 471 }, 472 .nb_slaves = 0, 473 .mode = CDEV_SCHED_MODE_NOT_SET, 474 .enable_ordering = 0, 475 .slave_names = { {0} } 476 }; 477 const char *name; 478 479 name = rte_vdev_device_name(vdev); 480 if (name == NULL) 481 return -EINVAL; 482 483 scheduler_parse_init_params(&init_params, 484 rte_vdev_device_args(vdev)); 485 486 487 return cryptodev_scheduler_create(name, 488 vdev, 489 &init_params); 490 } 491 492 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 493 .probe = cryptodev_scheduler_probe, 494 .remove = cryptodev_scheduler_remove 495 }; 496 497 static struct cryptodev_driver scheduler_crypto_drv; 498 499 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 500 cryptodev_scheduler_pmd_drv); 501 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 502 "max_nb_queue_pairs=<int> " 503 "max_nb_sessions=<int> " 504 "socket_id=<int> " 505 "slave=<name>"); 506 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 507 cryptodev_scheduler_pmd_drv.driver, 508 cryptodev_driver_id); 509