1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 #include <rte_common.h> 33 #include <rte_hexdump.h> 34 #include <rte_cryptodev.h> 35 #include <rte_cryptodev_pmd.h> 36 #include <rte_bus_vdev.h> 37 #include <rte_malloc.h> 38 #include <rte_cpuflags.h> 39 #include <rte_reorder.h> 40 41 #include "rte_cryptodev_scheduler.h" 42 #include "scheduler_pmd_private.h" 43 44 uint8_t cryptodev_driver_id; 45 46 struct scheduler_init_params { 47 struct rte_cryptodev_pmd_init_params def_p; 48 uint32_t nb_slaves; 49 enum rte_cryptodev_scheduler_mode mode; 50 uint32_t enable_ordering; 51 uint64_t wcmask; 52 char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] 53 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; 54 }; 55 56 #define RTE_CRYPTODEV_VDEV_NAME ("name") 57 #define RTE_CRYPTODEV_VDEV_SLAVE ("slave") 58 #define RTE_CRYPTODEV_VDEV_MODE ("mode") 59 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") 60 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") 61 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG ("max_nb_sessions") 62 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") 63 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") 64 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") 65 66 const char *scheduler_valid_params[] = { 67 RTE_CRYPTODEV_VDEV_NAME, 68 RTE_CRYPTODEV_VDEV_SLAVE, 69 RTE_CRYPTODEV_VDEV_MODE, 70 RTE_CRYPTODEV_VDEV_ORDERING, 71 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 72 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, 73 RTE_CRYPTODEV_VDEV_SOCKET_ID, 74 RTE_CRYPTODEV_VDEV_COREMASK, 75 RTE_CRYPTODEV_VDEV_CORELIST 76 }; 77 78 struct scheduler_parse_map { 79 const char *name; 80 uint32_t val; 81 }; 82 83 const struct scheduler_parse_map scheduler_mode_map[] = { 84 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), 85 CDEV_SCHED_MODE_ROUNDROBIN}, 86 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), 87 CDEV_SCHED_MODE_PKT_SIZE_DISTR}, 88 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), 89 CDEV_SCHED_MODE_FAILOVER}, 90 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), 91 CDEV_SCHED_MODE_MULTICORE} 92 }; 93 94 const struct scheduler_parse_map scheduler_ordering_map[] = { 95 {"enable", 1}, 96 {"disable", 0} 97 }; 98 99 static int 100 cryptodev_scheduler_create(const char *name, 101 struct rte_vdev_device *vdev, 102 struct scheduler_init_params *init_params) 103 { 104 struct rte_cryptodev *dev; 105 struct scheduler_ctx *sched_ctx; 106 uint32_t i; 107 int ret; 108 109 dev = rte_cryptodev_pmd_create(name, &vdev->device, 110 &init_params->def_p); 111 if (dev == NULL) { 112 CS_LOG_ERR("driver %s: failed to create cryptodev vdev", 113 name); 114 return -EFAULT; 115 } 116 117 if (init_params->wcmask != 0) 118 RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", 119 init_params->wcmask); 120 121 dev->driver_id = cryptodev_driver_id; 122 dev->dev_ops = rte_crypto_scheduler_pmd_ops; 123 124 sched_ctx = dev->data->dev_private; 125 sched_ctx->max_nb_queue_pairs = 126 init_params->def_p.max_nb_queue_pairs; 127 128 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { 129 uint16_t i; 130 131 sched_ctx->nb_wc = 0; 132 133 for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) { 134 if (init_params->wcmask & (1ULL << i)) { 135 sched_ctx->wc_pool[sched_ctx->nb_wc++] = i; 136 RTE_LOG(INFO, PMD, 137 " Worker core[%u]=%u added\n", 138 sched_ctx->nb_wc-1, i); 139 } 140 } 141 } 142 143 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && 144 init_params->mode < CDEV_SCHED_MODE_COUNT) { 145 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, 146 init_params->mode); 147 if (ret < 0) { 148 rte_cryptodev_pmd_release_device(dev); 149 return ret; 150 } 151 152 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 153 if (scheduler_mode_map[i].val != sched_ctx->mode) 154 continue; 155 156 RTE_LOG(INFO, PMD, " Scheduling mode = %s\n", 157 scheduler_mode_map[i].name); 158 break; 159 } 160 } 161 162 sched_ctx->reordering_enabled = init_params->enable_ordering; 163 164 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 165 if (scheduler_ordering_map[i].val != 166 sched_ctx->reordering_enabled) 167 continue; 168 169 RTE_LOG(INFO, PMD, " Packet ordering = %s\n", 170 scheduler_ordering_map[i].name); 171 172 break; 173 } 174 175 for (i = 0; i < init_params->nb_slaves; i++) { 176 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] = 177 rte_zmalloc_socket( 178 NULL, 179 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, 180 SOCKET_ID_ANY); 181 182 if (!sched_ctx->init_slave_names[ 183 sched_ctx->nb_init_slaves]) { 184 CS_LOG_ERR("driver %s: Insufficient memory", 185 name); 186 return -ENOMEM; 187 } 188 189 strncpy(sched_ctx->init_slave_names[ 190 sched_ctx->nb_init_slaves], 191 init_params->slave_names[i], 192 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 193 194 sched_ctx->nb_init_slaves++; 195 } 196 197 /* 198 * Initialize capabilities structure as an empty structure, 199 * in case device information is requested when no slaves are attached 200 */ 201 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 202 sizeof(struct rte_cryptodev_capabilities), 203 0, SOCKET_ID_ANY); 204 205 if (!sched_ctx->capabilities) { 206 RTE_LOG(ERR, PMD, "Not enough memory for capability " 207 "information\n"); 208 return -ENOMEM; 209 } 210 211 return 0; 212 } 213 214 static int 215 cryptodev_scheduler_remove(struct rte_vdev_device *vdev) 216 { 217 const char *name; 218 struct rte_cryptodev *dev; 219 struct scheduler_ctx *sched_ctx; 220 221 if (vdev == NULL) 222 return -EINVAL; 223 224 name = rte_vdev_device_name(vdev); 225 dev = rte_cryptodev_pmd_get_named_dev(name); 226 if (dev == NULL) 227 return -EINVAL; 228 229 sched_ctx = dev->data->dev_private; 230 231 if (sched_ctx->nb_slaves) { 232 uint32_t i; 233 234 for (i = 0; i < sched_ctx->nb_slaves; i++) 235 rte_cryptodev_scheduler_slave_detach(dev->data->dev_id, 236 sched_ctx->slaves[i].dev_id); 237 } 238 239 return rte_cryptodev_pmd_destroy(dev); 240 } 241 242 /** Parse integer from integer argument */ 243 static int 244 parse_integer_arg(const char *key __rte_unused, 245 const char *value, void *extra_args) 246 { 247 int *i = (int *) extra_args; 248 249 *i = atoi(value); 250 if (*i < 0) { 251 CS_LOG_ERR("Argument has to be positive.\n"); 252 return -EINVAL; 253 } 254 255 return 0; 256 } 257 258 /** Parse integer from hexadecimal integer argument */ 259 static int 260 parse_coremask_arg(const char *key __rte_unused, 261 const char *value, void *extra_args) 262 { 263 struct scheduler_init_params *params = extra_args; 264 265 params->wcmask = strtoull(value, NULL, 16); 266 267 return 0; 268 } 269 270 /** Parse integer from list of integers argument */ 271 static int 272 parse_corelist_arg(const char *key __rte_unused, 273 const char *value, void *extra_args) 274 { 275 struct scheduler_init_params *params = extra_args; 276 277 params->wcmask = 0ULL; 278 279 const char *token = value; 280 281 while (isdigit(token[0])) { 282 char *rval; 283 unsigned int core = strtoul(token, &rval, 10); 284 285 params->wcmask |= 1ULL << core; 286 token = (const char *)rval; 287 if (token[0] == '\0') 288 break; 289 token++; 290 } 291 292 return 0; 293 } 294 295 /** Parse name */ 296 static int 297 parse_name_arg(const char *key __rte_unused, 298 const char *value, void *extra_args) 299 { 300 struct rte_cryptodev_pmd_init_params *params = extra_args; 301 302 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 303 CS_LOG_ERR("Invalid name %s, should be less than " 304 "%u bytes.\n", value, 305 RTE_CRYPTODEV_NAME_MAX_LEN - 1); 306 return -EINVAL; 307 } 308 309 strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); 310 311 return 0; 312 } 313 314 /** Parse slave */ 315 static int 316 parse_slave_arg(const char *key __rte_unused, 317 const char *value, void *extra_args) 318 { 319 struct scheduler_init_params *param = extra_args; 320 321 if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 322 CS_LOG_ERR("Too many slaves.\n"); 323 return -ENOMEM; 324 } 325 326 strncpy(param->slave_names[param->nb_slaves++], value, 327 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); 328 329 return 0; 330 } 331 332 static int 333 parse_mode_arg(const char *key __rte_unused, 334 const char *value, void *extra_args) 335 { 336 struct scheduler_init_params *param = extra_args; 337 uint32_t i; 338 339 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { 340 if (strcmp(value, scheduler_mode_map[i].name) == 0) { 341 param->mode = (enum rte_cryptodev_scheduler_mode) 342 scheduler_mode_map[i].val; 343 break; 344 } 345 } 346 347 if (i == RTE_DIM(scheduler_mode_map)) { 348 CS_LOG_ERR("Unrecognized input.\n"); 349 return -EINVAL; 350 } 351 352 return 0; 353 } 354 355 static int 356 parse_ordering_arg(const char *key __rte_unused, 357 const char *value, void *extra_args) 358 { 359 struct scheduler_init_params *param = extra_args; 360 uint32_t i; 361 362 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { 363 if (strcmp(value, scheduler_ordering_map[i].name) == 0) { 364 param->enable_ordering = 365 scheduler_ordering_map[i].val; 366 break; 367 } 368 } 369 370 if (i == RTE_DIM(scheduler_ordering_map)) { 371 CS_LOG_ERR("Unrecognized input.\n"); 372 return -EINVAL; 373 } 374 375 return 0; 376 } 377 378 static int 379 scheduler_parse_init_params(struct scheduler_init_params *params, 380 const char *input_args) 381 { 382 struct rte_kvargs *kvlist = NULL; 383 int ret = 0; 384 385 if (params == NULL) 386 return -EINVAL; 387 388 if (input_args) { 389 kvlist = rte_kvargs_parse(input_args, 390 scheduler_valid_params); 391 if (kvlist == NULL) 392 return -1; 393 394 ret = rte_kvargs_process(kvlist, 395 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, 396 &parse_integer_arg, 397 ¶ms->def_p.max_nb_queue_pairs); 398 if (ret < 0) 399 goto free_kvlist; 400 401 ret = rte_kvargs_process(kvlist, 402 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, 403 &parse_integer_arg, 404 ¶ms->def_p.max_nb_sessions); 405 if (ret < 0) 406 goto free_kvlist; 407 408 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, 409 &parse_integer_arg, 410 ¶ms->def_p.socket_id); 411 if (ret < 0) 412 goto free_kvlist; 413 414 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, 415 &parse_coremask_arg, 416 params); 417 if (ret < 0) 418 goto free_kvlist; 419 420 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, 421 &parse_corelist_arg, 422 params); 423 if (ret < 0) 424 goto free_kvlist; 425 426 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, 427 &parse_name_arg, 428 ¶ms->def_p); 429 if (ret < 0) 430 goto free_kvlist; 431 432 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE, 433 &parse_slave_arg, params); 434 if (ret < 0) 435 goto free_kvlist; 436 437 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, 438 &parse_mode_arg, params); 439 if (ret < 0) 440 goto free_kvlist; 441 442 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, 443 &parse_ordering_arg, params); 444 if (ret < 0) 445 goto free_kvlist; 446 } 447 448 free_kvlist: 449 rte_kvargs_free(kvlist); 450 return ret; 451 } 452 453 static int 454 cryptodev_scheduler_probe(struct rte_vdev_device *vdev) 455 { 456 struct scheduler_init_params init_params = { 457 .def_p = { 458 "", 459 sizeof(struct scheduler_ctx), 460 rte_socket_id(), 461 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 462 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS 463 }, 464 .nb_slaves = 0, 465 .mode = CDEV_SCHED_MODE_NOT_SET, 466 .enable_ordering = 0, 467 .slave_names = { {0} } 468 }; 469 const char *name; 470 471 name = rte_vdev_device_name(vdev); 472 if (name == NULL) 473 return -EINVAL; 474 475 scheduler_parse_init_params(&init_params, 476 rte_vdev_device_args(vdev)); 477 478 479 return cryptodev_scheduler_create(name, 480 vdev, 481 &init_params); 482 } 483 484 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { 485 .probe = cryptodev_scheduler_probe, 486 .remove = cryptodev_scheduler_remove 487 }; 488 489 static struct cryptodev_driver scheduler_crypto_drv; 490 491 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, 492 cryptodev_scheduler_pmd_drv); 493 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, 494 "max_nb_queue_pairs=<int> " 495 "max_nb_sessions=<int> " 496 "socket_id=<int> " 497 "slave=<name>"); 498 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, 499 cryptodev_scheduler_pmd_drv, 500 cryptodev_driver_id); 501