1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_string_fns.h> 5 #include <rte_reorder.h> 6 #include <rte_cryptodev.h> 7 #include <rte_cryptodev_pmd.h> 8 #include <rte_malloc.h> 9 10 #include "rte_cryptodev_scheduler.h" 11 #include "scheduler_pmd_private.h" 12 13 int scheduler_logtype_driver; 14 15 /** update the scheduler pmd's capability with attaching device's 16 * capability. 17 * For each device to be attached, the scheduler's capability should be 18 * the common capability set of all slaves 19 **/ 20 static uint32_t 21 sync_caps(struct rte_cryptodev_capabilities *caps, 22 uint32_t nb_caps, 23 const struct rte_cryptodev_capabilities *slave_caps) 24 { 25 uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0; 26 uint32_t i; 27 28 while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED) 29 nb_slave_caps++; 30 31 if (nb_caps == 0) { 32 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps); 33 return nb_slave_caps; 34 } 35 36 for (i = 0; i < sync_nb_caps; i++) { 37 struct rte_cryptodev_capabilities *cap = &caps[i]; 38 uint32_t j; 39 40 for (j = 0; j < nb_slave_caps; j++) { 41 const struct rte_cryptodev_capabilities *s_cap = 42 &slave_caps[j]; 43 44 if (s_cap->op != cap->op || s_cap->sym.xform_type != 45 cap->sym.xform_type) 46 continue; 47 48 if (s_cap->sym.xform_type == 49 RTE_CRYPTO_SYM_XFORM_AUTH) { 50 if (s_cap->sym.auth.algo != 51 cap->sym.auth.algo) 52 continue; 53 54 cap->sym.auth.digest_size.min = 55 s_cap->sym.auth.digest_size.min < 56 cap->sym.auth.digest_size.min ? 57 s_cap->sym.auth.digest_size.min : 58 cap->sym.auth.digest_size.min; 59 cap->sym.auth.digest_size.max = 60 s_cap->sym.auth.digest_size.max < 61 cap->sym.auth.digest_size.max ? 62 s_cap->sym.auth.digest_size.max : 63 cap->sym.auth.digest_size.max; 64 65 } 66 67 if (s_cap->sym.xform_type == 68 RTE_CRYPTO_SYM_XFORM_CIPHER) 69 if (s_cap->sym.cipher.algo != 70 cap->sym.cipher.algo) 71 continue; 72 73 /* no common cap found */ 74 break; 75 } 76 77 if (j < nb_slave_caps) 78 continue; 79 80 /* remove a uncommon cap from the array */ 81 for (j = i; j < sync_nb_caps - 1; j++) 82 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap)); 83 84 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap)); 85 sync_nb_caps--; 86 } 87 88 return sync_nb_caps; 89 } 90 91 static int 92 update_scheduler_capability(struct scheduler_ctx *sched_ctx) 93 { 94 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} }; 95 uint32_t nb_caps = 0, i; 96 97 if (sched_ctx->capabilities) { 98 rte_free(sched_ctx->capabilities); 99 sched_ctx->capabilities = NULL; 100 } 101 102 for (i = 0; i < sched_ctx->nb_slaves; i++) { 103 struct rte_cryptodev_info dev_info; 104 105 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 106 107 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities); 108 if (nb_caps == 0) 109 return -1; 110 } 111 112 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 113 sizeof(struct rte_cryptodev_capabilities) * 114 (nb_caps + 1), 0, SOCKET_ID_ANY); 115 if (!sched_ctx->capabilities) 116 return -ENOMEM; 117 118 rte_memcpy(sched_ctx->capabilities, tmp_caps, 119 sizeof(struct rte_cryptodev_capabilities) * nb_caps); 120 121 return 0; 122 } 123 124 static void 125 update_scheduler_feature_flag(struct rte_cryptodev *dev) 126 { 127 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 128 uint32_t i; 129 130 dev->feature_flags = 0; 131 132 for (i = 0; i < sched_ctx->nb_slaves; i++) { 133 struct rte_cryptodev_info dev_info; 134 135 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 136 137 dev->feature_flags |= dev_info.feature_flags; 138 } 139 } 140 141 static void 142 update_max_nb_qp(struct scheduler_ctx *sched_ctx) 143 { 144 uint32_t i; 145 uint32_t max_nb_qp; 146 147 if (!sched_ctx->nb_slaves) 148 return; 149 150 max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0; 151 152 for (i = 0; i < sched_ctx->nb_slaves; i++) { 153 struct rte_cryptodev_info dev_info; 154 155 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 156 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ? 157 dev_info.max_nb_queue_pairs : max_nb_qp; 158 } 159 160 sched_ctx->max_nb_queue_pairs = max_nb_qp; 161 } 162 163 /** Attach a device to the scheduler. */ 164 int 165 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) 166 { 167 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 168 struct scheduler_ctx *sched_ctx; 169 struct scheduler_slave *slave; 170 struct rte_cryptodev_info dev_info; 171 uint32_t i; 172 173 if (!dev) { 174 CR_SCHED_LOG(ERR, "Operation not supported"); 175 return -ENOTSUP; 176 } 177 178 if (dev->driver_id != cryptodev_scheduler_driver_id) { 179 CR_SCHED_LOG(ERR, "Operation not supported"); 180 return -ENOTSUP; 181 } 182 183 if (dev->data->dev_started) { 184 CR_SCHED_LOG(ERR, "Illegal operation"); 185 return -EBUSY; 186 } 187 188 sched_ctx = dev->data->dev_private; 189 if (sched_ctx->nb_slaves >= 190 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 191 CR_SCHED_LOG(ERR, "Too many slaves attached"); 192 return -ENOMEM; 193 } 194 195 for (i = 0; i < sched_ctx->nb_slaves; i++) 196 if (sched_ctx->slaves[i].dev_id == slave_id) { 197 CR_SCHED_LOG(ERR, "Slave already added"); 198 return -ENOTSUP; 199 } 200 201 slave = &sched_ctx->slaves[sched_ctx->nb_slaves]; 202 203 rte_cryptodev_info_get(slave_id, &dev_info); 204 205 slave->dev_id = slave_id; 206 slave->driver_id = dev_info.driver_id; 207 sched_ctx->nb_slaves++; 208 209 if (update_scheduler_capability(sched_ctx) < 0) { 210 slave->dev_id = 0; 211 slave->driver_id = 0; 212 sched_ctx->nb_slaves--; 213 214 CR_SCHED_LOG(ERR, "capabilities update failed"); 215 return -ENOTSUP; 216 } 217 218 update_scheduler_feature_flag(dev); 219 220 update_max_nb_qp(sched_ctx); 221 222 return 0; 223 } 224 225 int 226 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id) 227 { 228 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 229 struct scheduler_ctx *sched_ctx; 230 uint32_t i, slave_pos; 231 232 if (!dev) { 233 CR_SCHED_LOG(ERR, "Operation not supported"); 234 return -ENOTSUP; 235 } 236 237 if (dev->driver_id != cryptodev_scheduler_driver_id) { 238 CR_SCHED_LOG(ERR, "Operation not supported"); 239 return -ENOTSUP; 240 } 241 242 if (dev->data->dev_started) { 243 CR_SCHED_LOG(ERR, "Illegal operation"); 244 return -EBUSY; 245 } 246 247 sched_ctx = dev->data->dev_private; 248 249 for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++) 250 if (sched_ctx->slaves[slave_pos].dev_id == slave_id) 251 break; 252 if (slave_pos == sched_ctx->nb_slaves) { 253 CR_SCHED_LOG(ERR, "Cannot find slave"); 254 return -ENOTSUP; 255 } 256 257 if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) { 258 CR_SCHED_LOG(ERR, "Failed to detach slave"); 259 return -ENOTSUP; 260 } 261 262 for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) { 263 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1], 264 sizeof(struct scheduler_slave)); 265 } 266 memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0, 267 sizeof(struct scheduler_slave)); 268 sched_ctx->nb_slaves--; 269 270 if (update_scheduler_capability(sched_ctx) < 0) { 271 CR_SCHED_LOG(ERR, "capabilities update failed"); 272 return -ENOTSUP; 273 } 274 275 update_scheduler_feature_flag(dev); 276 277 update_max_nb_qp(sched_ctx); 278 279 return 0; 280 } 281 282 int 283 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, 284 enum rte_cryptodev_scheduler_mode mode) 285 { 286 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 287 struct scheduler_ctx *sched_ctx; 288 289 if (!dev) { 290 CR_SCHED_LOG(ERR, "Operation not supported"); 291 return -ENOTSUP; 292 } 293 294 if (dev->driver_id != cryptodev_scheduler_driver_id) { 295 CR_SCHED_LOG(ERR, "Operation not supported"); 296 return -ENOTSUP; 297 } 298 299 if (dev->data->dev_started) { 300 CR_SCHED_LOG(ERR, "Illegal operation"); 301 return -EBUSY; 302 } 303 304 sched_ctx = dev->data->dev_private; 305 306 if (mode == sched_ctx->mode) 307 return 0; 308 309 switch (mode) { 310 case CDEV_SCHED_MODE_ROUNDROBIN: 311 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 312 crypto_scheduler_roundrobin) < 0) { 313 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 314 return -1; 315 } 316 break; 317 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 318 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 319 crypto_scheduler_pkt_size_based_distr) < 0) { 320 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 321 return -1; 322 } 323 break; 324 case CDEV_SCHED_MODE_FAILOVER: 325 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 326 crypto_scheduler_failover) < 0) { 327 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 328 return -1; 329 } 330 break; 331 case CDEV_SCHED_MODE_MULTICORE: 332 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 333 crypto_scheduler_multicore) < 0) { 334 CR_SCHED_LOG(ERR, "Failed to load scheduler"); 335 return -1; 336 } 337 break; 338 default: 339 CR_SCHED_LOG(ERR, "Not yet supported"); 340 return -ENOTSUP; 341 } 342 343 return 0; 344 } 345 346 enum rte_cryptodev_scheduler_mode 347 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) 348 { 349 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 350 struct scheduler_ctx *sched_ctx; 351 352 if (!dev) { 353 CR_SCHED_LOG(ERR, "Operation not supported"); 354 return -ENOTSUP; 355 } 356 357 if (dev->driver_id != cryptodev_scheduler_driver_id) { 358 CR_SCHED_LOG(ERR, "Operation not supported"); 359 return -ENOTSUP; 360 } 361 362 sched_ctx = dev->data->dev_private; 363 364 return sched_ctx->mode; 365 } 366 367 int 368 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, 369 uint32_t enable_reorder) 370 { 371 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 372 struct scheduler_ctx *sched_ctx; 373 374 if (!dev) { 375 CR_SCHED_LOG(ERR, "Operation not supported"); 376 return -ENOTSUP; 377 } 378 379 if (dev->driver_id != cryptodev_scheduler_driver_id) { 380 CR_SCHED_LOG(ERR, "Operation not supported"); 381 return -ENOTSUP; 382 } 383 384 if (dev->data->dev_started) { 385 CR_SCHED_LOG(ERR, "Illegal operation"); 386 return -EBUSY; 387 } 388 389 sched_ctx = dev->data->dev_private; 390 391 sched_ctx->reordering_enabled = enable_reorder; 392 393 return 0; 394 } 395 396 int 397 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) 398 { 399 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 400 struct scheduler_ctx *sched_ctx; 401 402 if (!dev) { 403 CR_SCHED_LOG(ERR, "Operation not supported"); 404 return -ENOTSUP; 405 } 406 407 if (dev->driver_id != cryptodev_scheduler_driver_id) { 408 CR_SCHED_LOG(ERR, "Operation not supported"); 409 return -ENOTSUP; 410 } 411 412 sched_ctx = dev->data->dev_private; 413 414 return (int)sched_ctx->reordering_enabled; 415 } 416 417 int 418 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, 419 struct rte_cryptodev_scheduler *scheduler) { 420 421 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 422 struct scheduler_ctx *sched_ctx; 423 424 if (!dev) { 425 CR_SCHED_LOG(ERR, "Operation not supported"); 426 return -ENOTSUP; 427 } 428 429 if (dev->driver_id != cryptodev_scheduler_driver_id) { 430 CR_SCHED_LOG(ERR, "Operation not supported"); 431 return -ENOTSUP; 432 } 433 434 if (dev->data->dev_started) { 435 CR_SCHED_LOG(ERR, "Illegal operation"); 436 return -EBUSY; 437 } 438 439 sched_ctx = dev->data->dev_private; 440 441 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 442 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than " 443 "%u bytes.", scheduler->name, 444 RTE_CRYPTODEV_NAME_MAX_LEN); 445 return -EINVAL; 446 } 447 strlcpy(sched_ctx->name, scheduler->name, sizeof(sched_ctx->name)); 448 449 if (strlen(scheduler->description) > 450 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { 451 CR_SCHED_LOG(ERR, "Invalid description %s, should be less than " 452 "%u bytes.", scheduler->description, 453 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); 454 return -EINVAL; 455 } 456 strlcpy(sched_ctx->description, scheduler->description, 457 sizeof(sched_ctx->description)); 458 459 /* load scheduler instance operations functions */ 460 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair; 461 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx; 462 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start; 463 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop; 464 sched_ctx->ops.slave_attach = scheduler->ops->slave_attach; 465 sched_ctx->ops.slave_detach = scheduler->ops->slave_detach; 466 sched_ctx->ops.option_set = scheduler->ops->option_set; 467 sched_ctx->ops.option_get = scheduler->ops->option_get; 468 469 if (sched_ctx->private_ctx) { 470 rte_free(sched_ctx->private_ctx); 471 sched_ctx->private_ctx = NULL; 472 } 473 474 if (sched_ctx->ops.create_private_ctx) { 475 int ret = (*sched_ctx->ops.create_private_ctx)(dev); 476 477 if (ret < 0) { 478 CR_SCHED_LOG(ERR, "Unable to create scheduler private " 479 "context"); 480 return ret; 481 } 482 } 483 484 sched_ctx->mode = scheduler->mode; 485 486 return 0; 487 } 488 489 int 490 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves) 491 { 492 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 493 struct scheduler_ctx *sched_ctx; 494 uint32_t nb_slaves = 0; 495 496 if (!dev) { 497 CR_SCHED_LOG(ERR, "Operation not supported"); 498 return -ENOTSUP; 499 } 500 501 if (dev->driver_id != cryptodev_scheduler_driver_id) { 502 CR_SCHED_LOG(ERR, "Operation not supported"); 503 return -ENOTSUP; 504 } 505 506 sched_ctx = dev->data->dev_private; 507 508 nb_slaves = sched_ctx->nb_slaves; 509 510 if (slaves && nb_slaves) { 511 uint32_t i; 512 513 for (i = 0; i < nb_slaves; i++) 514 slaves[i] = sched_ctx->slaves[i].dev_id; 515 } 516 517 return (int)nb_slaves; 518 } 519 520 int 521 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id, 522 enum rte_cryptodev_schedule_option_type option_type, 523 void *option) 524 { 525 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 526 struct scheduler_ctx *sched_ctx; 527 528 if (option_type == CDEV_SCHED_OPTION_NOT_SET || 529 option_type >= CDEV_SCHED_OPTION_COUNT) { 530 CR_SCHED_LOG(ERR, "Invalid option parameter"); 531 return -EINVAL; 532 } 533 534 if (!option) { 535 CR_SCHED_LOG(ERR, "Invalid option parameter"); 536 return -EINVAL; 537 } 538 539 if (dev->data->dev_started) { 540 CR_SCHED_LOG(ERR, "Illegal operation"); 541 return -EBUSY; 542 } 543 544 sched_ctx = dev->data->dev_private; 545 546 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP); 547 548 return (*sched_ctx->ops.option_set)(dev, option_type, option); 549 } 550 551 int 552 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, 553 enum rte_cryptodev_schedule_option_type option_type, 554 void *option) 555 { 556 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 557 struct scheduler_ctx *sched_ctx; 558 559 if (!dev) { 560 CR_SCHED_LOG(ERR, "Operation not supported"); 561 return -ENOTSUP; 562 } 563 564 if (!option) { 565 CR_SCHED_LOG(ERR, "Invalid option parameter"); 566 return -EINVAL; 567 } 568 569 if (dev->driver_id != cryptodev_scheduler_driver_id) { 570 CR_SCHED_LOG(ERR, "Operation not supported"); 571 return -ENOTSUP; 572 } 573 574 sched_ctx = dev->data->dev_private; 575 576 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP); 577 578 return (*sched_ctx->ops.option_get)(dev, option_type, option); 579 } 580 581 RTE_INIT(scheduler_init_log) 582 { 583 scheduler_logtype_driver = rte_log_register("pmd.crypto.scheduler"); 584 } 585