1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_string_fns.h> 5 #include <rte_reorder.h> 6 #include <rte_cryptodev.h> 7 #include <cryptodev_pmd.h> 8 #include <rte_malloc.h> 9 10 #include "rte_cryptodev_scheduler.h" 11 #include "scheduler_pmd_private.h" 12 13 /** update the scheduler pmd's capability with attaching device's 14 * capability. 15 * For each device to be attached, the scheduler's capability should be 16 * the common capability set of all workers 17 **/ 18 static uint32_t 19 sync_caps(struct rte_cryptodev_capabilities *caps, 20 uint32_t nb_caps, 21 const struct rte_cryptodev_capabilities *worker_caps) 22 { 23 uint32_t sync_nb_caps = nb_caps, nb_worker_caps = 0; 24 uint32_t i; 25 26 while (worker_caps[nb_worker_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED) 27 nb_worker_caps++; 28 29 if (nb_caps == 0) { 30 rte_memcpy(caps, worker_caps, sizeof(*caps) * nb_worker_caps); 31 return nb_worker_caps; 32 } 33 34 for (i = 0; i < sync_nb_caps; i++) { 35 struct rte_cryptodev_capabilities *cap = &caps[i]; 36 uint32_t j; 37 38 for (j = 0; j < nb_worker_caps; j++) { 39 const struct rte_cryptodev_capabilities *s_cap = 40 &worker_caps[j]; 41 42 if (s_cap->op != cap->op || s_cap->sym.xform_type != 43 cap->sym.xform_type) 44 continue; 45 46 if (s_cap->sym.xform_type == 47 RTE_CRYPTO_SYM_XFORM_AUTH) { 48 if (s_cap->sym.auth.algo != 49 cap->sym.auth.algo) 50 continue; 51 52 cap->sym.auth.digest_size.min = 53 s_cap->sym.auth.digest_size.min < 54 cap->sym.auth.digest_size.min ? 55 s_cap->sym.auth.digest_size.min : 56 cap->sym.auth.digest_size.min; 57 cap->sym.auth.digest_size.max = 58 s_cap->sym.auth.digest_size.max < 59 cap->sym.auth.digest_size.max ? 60 s_cap->sym.auth.digest_size.max : 61 cap->sym.auth.digest_size.max; 62 63 } 64 65 if (s_cap->sym.xform_type == 66 RTE_CRYPTO_SYM_XFORM_CIPHER) 67 if (s_cap->sym.cipher.algo != 68 cap->sym.cipher.algo) 69 continue; 70 71 /* no common cap found */ 72 break; 73 } 74 75 if (j < nb_worker_caps) 76 continue; 77 78 /* remove a uncommon cap from the array */ 79 for (j = i; j < sync_nb_caps - 1; j++) 80 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap)); 81 82 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap)); 83 sync_nb_caps--; 84 } 85 86 return sync_nb_caps; 87 } 88 89 static int 90 update_scheduler_capability(struct scheduler_ctx *sched_ctx) 91 { 92 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} }; 93 uint32_t nb_caps = 0, i; 94 95 if (sched_ctx->capabilities) { 96 rte_free(sched_ctx->capabilities); 97 sched_ctx->capabilities = NULL; 98 } 99 100 for (i = 0; i < sched_ctx->nb_workers; i++) { 101 struct rte_cryptodev_info dev_info; 102 103 rte_cryptodev_info_get(sched_ctx->workers[i].dev_id, &dev_info); 104 105 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities); 106 if (nb_caps == 0) 107 return -1; 108 } 109 110 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 111 sizeof(struct rte_cryptodev_capabilities) * 112 (nb_caps + 1), 0, SOCKET_ID_ANY); 113 if (!sched_ctx->capabilities) 114 return -ENOMEM; 115 116 rte_memcpy(sched_ctx->capabilities, tmp_caps, 117 sizeof(struct rte_cryptodev_capabilities) * nb_caps); 118 119 return 0; 120 } 121 122 static void 123 update_scheduler_feature_flag(struct rte_cryptodev *dev) 124 { 125 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 126 uint32_t i; 127 128 dev->feature_flags = 0; 129 130 for (i = 0; i < sched_ctx->nb_workers; i++) { 131 struct rte_cryptodev_info dev_info; 132 133 rte_cryptodev_info_get(sched_ctx->workers[i].dev_id, &dev_info); 134 135 dev->feature_flags |= dev_info.feature_flags; 136 } 137 } 138 139 static void 140 update_max_nb_qp(struct scheduler_ctx *sched_ctx) 141 { 142 uint32_t i; 143 uint32_t max_nb_qp; 144 145 if (!sched_ctx->nb_workers) 146 return; 147 148 max_nb_qp = sched_ctx->nb_workers ? UINT32_MAX : 0; 149 150 for (i = 0; i < sched_ctx->nb_workers; i++) { 151 struct rte_cryptodev_info dev_info; 152 153 rte_cryptodev_info_get(sched_ctx->workers[i].dev_id, &dev_info); 154 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ? 155 dev_info.max_nb_queue_pairs : max_nb_qp; 156 } 157 158 sched_ctx->max_nb_queue_pairs = max_nb_qp; 159 } 160 161 /** Attach a device to the scheduler. */ 162 int 163 rte_cryptodev_scheduler_worker_attach(uint8_t scheduler_id, uint8_t worker_id) 164 { 165 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 166 struct scheduler_ctx *sched_ctx; 167 struct scheduler_worker *worker; 168 struct rte_cryptodev_info dev_info; 169 uint32_t i; 170 171 if (!dev) { 172 CR_SCHED_LOG(ERR, "Operation not supported"); 173 return -ENOTSUP; 174 } 175 176 if (dev->driver_id != cryptodev_scheduler_driver_id) { 177 CR_SCHED_LOG(ERR, "Operation not supported"); 178 return -ENOTSUP; 179 } 180 181 if (dev->data->dev_started) { 182 CR_SCHED_LOG(ERR, "Illegal operation"); 183 return -EBUSY; 184 } 185 186 sched_ctx = dev->data->dev_private; 187 if (sched_ctx->nb_workers >= 188 RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) { 189 CR_SCHED_LOG(ERR, "Too many workers attached"); 190 return -ENOMEM; 191 } 192 193 for (i = 0; i < sched_ctx->nb_workers; i++) 194 if (sched_ctx->workers[i].dev_id == worker_id) { 195 CR_SCHED_LOG(ERR, "Worker already added"); 196 return -ENOTSUP; 197 } 198 199 worker = &sched_ctx->workers[sched_ctx->nb_workers]; 200 201 rte_cryptodev_info_get(worker_id, &dev_info); 202 203 worker->dev_id = worker_id; 204 worker->driver_id = dev_info.driver_id; 205 sched_ctx->nb_workers++; 206 207 if (update_scheduler_capability(sched_ctx) < 0) { 208 worker->dev_id = 0; 209 worker->driver_id = 0; 210 sched_ctx->nb_workers--; 211 212 CR_SCHED_LOG(ERR, "capabilities update failed"); 213 return -ENOTSUP; 214 } 215 216 update_scheduler_feature_flag(dev); 217 218 update_max_nb_qp(sched_ctx); 219 220 return 0; 221 } 222 223 int 224 rte_cryptodev_scheduler_worker_detach(uint8_t scheduler_id, uint8_t worker_id) 225 { 226 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 227 struct scheduler_ctx *sched_ctx; 228 uint32_t i, worker_pos; 229 230 if (!dev) { 231 CR_SCHED_LOG(ERR, "Operation not supported"); 232 return -ENOTSUP; 233 } 234 235 if (dev->driver_id != cryptodev_scheduler_driver_id) { 236 CR_SCHED_LOG(ERR, "Operation not supported"); 237 return -ENOTSUP; 238 } 239 240 if (dev->data->dev_started) { 241 CR_SCHED_LOG(ERR, "Illegal operation"); 242 return -EBUSY; 243 } 244 245 sched_ctx = dev->data->dev_private; 246 247 for (worker_pos = 0; worker_pos < sched_ctx->nb_workers; worker_pos++) 248 if (sched_ctx->workers[worker_pos].dev_id == worker_id) 249 break; 250 if (worker_pos == sched_ctx->nb_workers) { 251 CR_SCHED_LOG(ERR, "Cannot find worker"); 252 return -ENOTSUP; 253 } 254 255 if (sched_ctx->ops.worker_detach(dev, worker_id) < 0) { 256 CR_SCHED_LOG(ERR, "Failed to detach worker"); 257 return -ENOTSUP; 258 } 259 260 for (i = worker_pos; i < sched_ctx->nb_workers - 1; i++) { 261 memcpy(&sched_ctx->workers[i], &sched_ctx->workers[i+1], 262 sizeof(struct scheduler_worker)); 263 } 264 memset(&sched_ctx->workers[sched_ctx->nb_workers - 1], 0, 265 sizeof(struct scheduler_worker)); 266 sched_ctx->nb_workers--; 267 268 if (update_scheduler_capability(sched_ctx) < 0) { 269 CR_SCHED_LOG(ERR, "capabilities update failed"); 270 return -ENOTSUP; 271 } 272 273 update_scheduler_feature_flag(dev); 274 275 update_max_nb_qp(sched_ctx); 276 277 return 0; 278 } 279 280 int 281 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, 282 enum rte_cryptodev_scheduler_mode mode) 283 { 284 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 285 struct scheduler_ctx *sched_ctx; 286 287 if (!dev) { 288 CR_SCHED_LOG(ERR, "Operation not supported"); 289 return -ENOTSUP; 290 } 291 292 if (dev->driver_id != cryptodev_scheduler_driver_id) { 293 CR_SCHED_LOG(ERR, "Operation not supported"); 294 return -ENOTSUP; 295 } 296 297 if (dev->data->dev_started) { 298 CR_SCHED_LOG(ERR, "Illegal operation"); 299 return -EBUSY; 300 } 301 302 sched_ctx = dev->data->dev_private; 303 304 if (mode == sched_ctx->mode) 305 return 0; 306 307 switch (mode) { 308 case CDEV_SCHED_MODE_ROUNDROBIN: 309 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 310 crypto_scheduler_roundrobin) < 0) { 311 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 312 return -1; 313 } 314 break; 315 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 316 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 317 crypto_scheduler_pkt_size_based_distr) < 0) { 318 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 319 return -1; 320 } 321 break; 322 case CDEV_SCHED_MODE_FAILOVER: 323 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 324 crypto_scheduler_failover) < 0) { 325 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 326 return -1; 327 } 328 break; 329 case CDEV_SCHED_MODE_MULTICORE: 330 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 331 crypto_scheduler_multicore) < 0) { 332 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 333 return -1; 334 } 335 break; 336 default: 337 CR_SCHED_LOG(ERR, "Not yet supported"); 338 return -ENOTSUP; 339 } 340 341 return 0; 342 } 343 344 enum rte_cryptodev_scheduler_mode 345 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) 346 { 347 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 348 struct scheduler_ctx *sched_ctx; 349 350 if (!dev) { 351 CR_SCHED_LOG(ERR, "Operation not supported"); 352 return -ENOTSUP; 353 } 354 355 if (dev->driver_id != cryptodev_scheduler_driver_id) { 356 CR_SCHED_LOG(ERR, "Operation not supported"); 357 return -ENOTSUP; 358 } 359 360 sched_ctx = dev->data->dev_private; 361 362 return sched_ctx->mode; 363 } 364 365 int 366 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, 367 uint32_t enable_reorder) 368 { 369 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 370 struct scheduler_ctx *sched_ctx; 371 372 if (!dev) { 373 CR_SCHED_LOG(ERR, "Operation not supported"); 374 return -ENOTSUP; 375 } 376 377 if (dev->driver_id != cryptodev_scheduler_driver_id) { 378 CR_SCHED_LOG(ERR, "Operation not supported"); 379 return -ENOTSUP; 380 } 381 382 if (dev->data->dev_started) { 383 CR_SCHED_LOG(ERR, "Illegal operation"); 384 return -EBUSY; 385 } 386 387 sched_ctx = dev->data->dev_private; 388 389 sched_ctx->reordering_enabled = enable_reorder; 390 391 return 0; 392 } 393 394 int 395 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) 396 { 397 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 398 struct scheduler_ctx *sched_ctx; 399 400 if (!dev) { 401 CR_SCHED_LOG(ERR, "Operation not supported"); 402 return -ENOTSUP; 403 } 404 405 if (dev->driver_id != cryptodev_scheduler_driver_id) { 406 CR_SCHED_LOG(ERR, "Operation not supported"); 407 return -ENOTSUP; 408 } 409 410 sched_ctx = dev->data->dev_private; 411 412 return (int)sched_ctx->reordering_enabled; 413 } 414 415 int 416 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, 417 struct rte_cryptodev_scheduler *scheduler) { 418 419 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 420 struct scheduler_ctx *sched_ctx; 421 422 if (!dev) { 423 CR_SCHED_LOG(ERR, "Operation not supported"); 424 return -ENOTSUP; 425 } 426 427 if (dev->driver_id != cryptodev_scheduler_driver_id) { 428 CR_SCHED_LOG(ERR, "Operation not supported"); 429 return -ENOTSUP; 430 } 431 432 if (dev->data->dev_started) { 433 CR_SCHED_LOG(ERR, "Illegal operation"); 434 return -EBUSY; 435 } 436 437 sched_ctx = dev->data->dev_private; 438 439 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 440 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 441 "%u bytes.", scheduler->name, 442 RTE_CRYPTODEV_NAME_MAX_LEN); 443 return -EINVAL; 444 } 445 strlcpy(sched_ctx->name, scheduler->name, sizeof(sched_ctx->name)); 446 447 if (strlen(scheduler->description) > 448 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { 449 CR_SCHED_LOG(ERR, "Invalid description %s, should be less than " 450 "%u bytes.", scheduler->description, 451 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); 452 return -EINVAL; 453 } 454 strlcpy(sched_ctx->description, scheduler->description, 455 sizeof(sched_ctx->description)); 456 457 /* load scheduler instance operations functions */ 458 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair; 459 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx; 460 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start; 461 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop; 462 sched_ctx->ops.worker_attach = scheduler->ops->worker_attach; 463 sched_ctx->ops.worker_detach = scheduler->ops->worker_detach; 464 sched_ctx->ops.option_set = scheduler->ops->option_set; 465 sched_ctx->ops.option_get = scheduler->ops->option_get; 466 467 if (sched_ctx->private_ctx) { 468 rte_free(sched_ctx->private_ctx); 469 sched_ctx->private_ctx = NULL; 470 } 471 472 if (sched_ctx->ops.create_private_ctx) { 473 int ret = (*sched_ctx->ops.create_private_ctx)(dev); 474 475 if (ret < 0) { 476 CR_SCHED_LOG(ERR, "Unable to create scheduler private " 477 "context"); 478 return ret; 479 } 480 } 481 482 sched_ctx->mode = scheduler->mode; 483 484 return 0; 485 } 486 487 int 488 rte_cryptodev_scheduler_workers_get(uint8_t scheduler_id, uint8_t *workers) 489 { 490 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 491 struct scheduler_ctx *sched_ctx; 492 uint32_t nb_workers = 0; 493 494 if (!dev) { 495 CR_SCHED_LOG(ERR, "Operation not supported"); 496 return -ENOTSUP; 497 } 498 499 if (dev->driver_id != cryptodev_scheduler_driver_id) { 500 CR_SCHED_LOG(ERR, "Operation not supported"); 501 return -ENOTSUP; 502 } 503 504 sched_ctx = dev->data->dev_private; 505 506 nb_workers = sched_ctx->nb_workers; 507 508 if (workers && nb_workers) { 509 uint32_t i; 510 511 for (i = 0; i < nb_workers; i++) 512 workers[i] = sched_ctx->workers[i].dev_id; 513 } 514 515 return (int)nb_workers; 516 } 517 518 int 519 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id, 520 enum rte_cryptodev_schedule_option_type option_type, 521 void *option) 522 { 523 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 524 struct scheduler_ctx *sched_ctx; 525 526 if (option_type == CDEV_SCHED_OPTION_NOT_SET || 527 option_type >= CDEV_SCHED_OPTION_COUNT) { 528 CR_SCHED_LOG(ERR, "Invalid option parameter"); 529 return -EINVAL; 530 } 531 532 if (!option) { 533 CR_SCHED_LOG(ERR, "Invalid option parameter"); 534 return -EINVAL; 535 } 536 537 if (dev->data->dev_started) { 538 CR_SCHED_LOG(ERR, "Illegal operation"); 539 return -EBUSY; 540 } 541 542 sched_ctx = dev->data->dev_private; 543 544 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP); 545 546 return (*sched_ctx->ops.option_set)(dev, option_type, option); 547 } 548 549 int 550 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, 551 enum rte_cryptodev_schedule_option_type option_type, 552 void *option) 553 { 554 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 555 struct scheduler_ctx *sched_ctx; 556 557 if (!dev) { 558 CR_SCHED_LOG(ERR, "Operation not supported"); 559 return -ENOTSUP; 560 } 561 562 if (!option) { 563 CR_SCHED_LOG(ERR, "Invalid option parameter"); 564 return -EINVAL; 565 } 566 567 if (dev->driver_id != cryptodev_scheduler_driver_id) { 568 CR_SCHED_LOG(ERR, "Operation not supported"); 569 return -ENOTSUP; 570 } 571 572 sched_ctx = dev->data->dev_private; 573 574 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP); 575 576 return (*sched_ctx->ops.option_get)(dev, option_type, option); 577 } 578 579 580 RTE_LOG_REGISTER_DEFAULT(scheduler_logtype_driver, INFO); 581