1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 #include <rte_reorder.h> 5 #include <rte_cryptodev.h> 6 #include <rte_cryptodev_pmd.h> 7 #include <rte_malloc.h> 8 9 #include "rte_cryptodev_scheduler.h" 10 #include "scheduler_pmd_private.h" 11 12 /** update the scheduler pmd's capability with attaching device's 13 * capability. 14 * For each device to be attached, the scheduler's capability should be 15 * the common capability set of all slaves 16 **/ 17 static uint32_t 18 sync_caps(struct rte_cryptodev_capabilities *caps, 19 uint32_t nb_caps, 20 const struct rte_cryptodev_capabilities *slave_caps) 21 { 22 uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0; 23 uint32_t i; 24 25 while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED) 26 nb_slave_caps++; 27 28 if (nb_caps == 0) { 29 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps); 30 return nb_slave_caps; 31 } 32 33 for (i = 0; i < sync_nb_caps; i++) { 34 struct rte_cryptodev_capabilities *cap = &caps[i]; 35 uint32_t j; 36 37 for (j = 0; j < nb_slave_caps; j++) { 38 const struct rte_cryptodev_capabilities *s_cap = 39 &slave_caps[j]; 40 41 if (s_cap->op != cap->op || s_cap->sym.xform_type != 42 cap->sym.xform_type) 43 continue; 44 45 if (s_cap->sym.xform_type == 46 RTE_CRYPTO_SYM_XFORM_AUTH) { 47 if (s_cap->sym.auth.algo != 48 cap->sym.auth.algo) 49 continue; 50 51 cap->sym.auth.digest_size.min = 52 s_cap->sym.auth.digest_size.min < 53 cap->sym.auth.digest_size.min ? 54 s_cap->sym.auth.digest_size.min : 55 cap->sym.auth.digest_size.min; 56 cap->sym.auth.digest_size.max = 57 s_cap->sym.auth.digest_size.max < 58 cap->sym.auth.digest_size.max ? 59 s_cap->sym.auth.digest_size.max : 60 cap->sym.auth.digest_size.max; 61 62 } 63 64 if (s_cap->sym.xform_type == 65 RTE_CRYPTO_SYM_XFORM_CIPHER) 66 if (s_cap->sym.cipher.algo != 67 cap->sym.cipher.algo) 68 continue; 69 70 /* no common cap found */ 71 break; 72 } 73 74 if (j < nb_slave_caps) 75 continue; 76 77 /* remove a uncommon cap from the array */ 78 for (j = i; j < sync_nb_caps - 1; j++) 79 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap)); 80 81 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap)); 82 sync_nb_caps--; 83 } 84 85 return sync_nb_caps; 86 } 87 88 static int 89 update_scheduler_capability(struct scheduler_ctx *sched_ctx) 90 { 91 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} }; 92 uint32_t nb_caps = 0, i; 93 94 if (sched_ctx->capabilities) 95 rte_free(sched_ctx->capabilities); 96 97 for (i = 0; i < sched_ctx->nb_slaves; i++) { 98 struct rte_cryptodev_info dev_info; 99 100 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 101 102 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities); 103 if (nb_caps == 0) 104 return -1; 105 } 106 107 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 108 sizeof(struct rte_cryptodev_capabilities) * 109 (nb_caps + 1), 0, SOCKET_ID_ANY); 110 if (!sched_ctx->capabilities) 111 return -ENOMEM; 112 113 rte_memcpy(sched_ctx->capabilities, tmp_caps, 114 sizeof(struct rte_cryptodev_capabilities) * nb_caps); 115 116 return 0; 117 } 118 119 static void 120 update_scheduler_feature_flag(struct rte_cryptodev *dev) 121 { 122 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 123 uint32_t i; 124 125 dev->feature_flags = 0; 126 127 for (i = 0; i < sched_ctx->nb_slaves; i++) { 128 struct rte_cryptodev_info dev_info; 129 130 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 131 132 dev->feature_flags |= dev_info.feature_flags; 133 } 134 } 135 136 static void 137 update_max_nb_qp(struct scheduler_ctx *sched_ctx) 138 { 139 uint32_t i; 140 uint32_t max_nb_qp; 141 142 if (!sched_ctx->nb_slaves) 143 return; 144 145 max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0; 146 147 for (i = 0; i < sched_ctx->nb_slaves; i++) { 148 struct rte_cryptodev_info dev_info; 149 150 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 151 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ? 152 dev_info.max_nb_queue_pairs : max_nb_qp; 153 } 154 155 sched_ctx->max_nb_queue_pairs = max_nb_qp; 156 } 157 158 /** Attach a device to the scheduler. */ 159 int 160 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) 161 { 162 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 163 struct scheduler_ctx *sched_ctx; 164 struct scheduler_slave *slave; 165 struct rte_cryptodev_info dev_info; 166 uint32_t i; 167 168 if (!dev) { 169 CS_LOG_ERR("Operation not supported"); 170 return -ENOTSUP; 171 } 172 173 if (dev->driver_id != cryptodev_driver_id) { 174 CS_LOG_ERR("Operation not supported"); 175 return -ENOTSUP; 176 } 177 178 if (dev->data->dev_started) { 179 CS_LOG_ERR("Illegal operation"); 180 return -EBUSY; 181 } 182 183 sched_ctx = dev->data->dev_private; 184 if (sched_ctx->nb_slaves >= 185 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 186 CS_LOG_ERR("Too many slaves attached"); 187 return -ENOMEM; 188 } 189 190 for (i = 0; i < sched_ctx->nb_slaves; i++) 191 if (sched_ctx->slaves[i].dev_id == slave_id) { 192 CS_LOG_ERR("Slave already added"); 193 return -ENOTSUP; 194 } 195 196 slave = &sched_ctx->slaves[sched_ctx->nb_slaves]; 197 198 rte_cryptodev_info_get(slave_id, &dev_info); 199 200 slave->dev_id = slave_id; 201 slave->driver_id = dev_info.driver_id; 202 sched_ctx->nb_slaves++; 203 204 if (update_scheduler_capability(sched_ctx) < 0) { 205 slave->dev_id = 0; 206 slave->driver_id = 0; 207 sched_ctx->nb_slaves--; 208 209 CS_LOG_ERR("capabilities update failed"); 210 return -ENOTSUP; 211 } 212 213 update_scheduler_feature_flag(dev); 214 215 update_max_nb_qp(sched_ctx); 216 217 return 0; 218 } 219 220 int 221 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id) 222 { 223 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 224 struct scheduler_ctx *sched_ctx; 225 uint32_t i, slave_pos; 226 227 if (!dev) { 228 CS_LOG_ERR("Operation not supported"); 229 return -ENOTSUP; 230 } 231 232 if (dev->driver_id != cryptodev_driver_id) { 233 CS_LOG_ERR("Operation not supported"); 234 return -ENOTSUP; 235 } 236 237 if (dev->data->dev_started) { 238 CS_LOG_ERR("Illegal operation"); 239 return -EBUSY; 240 } 241 242 sched_ctx = dev->data->dev_private; 243 244 for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++) 245 if (sched_ctx->slaves[slave_pos].dev_id == slave_id) 246 break; 247 if (slave_pos == sched_ctx->nb_slaves) { 248 CS_LOG_ERR("Cannot find slave"); 249 return -ENOTSUP; 250 } 251 252 if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) { 253 CS_LOG_ERR("Failed to detach slave"); 254 return -ENOTSUP; 255 } 256 257 for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) { 258 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1], 259 sizeof(struct scheduler_slave)); 260 } 261 memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0, 262 sizeof(struct scheduler_slave)); 263 sched_ctx->nb_slaves--; 264 265 if (update_scheduler_capability(sched_ctx) < 0) { 266 CS_LOG_ERR("capabilities update failed"); 267 return -ENOTSUP; 268 } 269 270 update_scheduler_feature_flag(dev); 271 272 update_max_nb_qp(sched_ctx); 273 274 return 0; 275 } 276 277 int 278 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, 279 enum rte_cryptodev_scheduler_mode mode) 280 { 281 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 282 struct scheduler_ctx *sched_ctx; 283 284 if (!dev) { 285 CS_LOG_ERR("Operation not supported"); 286 return -ENOTSUP; 287 } 288 289 if (dev->driver_id != cryptodev_driver_id) { 290 CS_LOG_ERR("Operation not supported"); 291 return -ENOTSUP; 292 } 293 294 if (dev->data->dev_started) { 295 CS_LOG_ERR("Illegal operation"); 296 return -EBUSY; 297 } 298 299 sched_ctx = dev->data->dev_private; 300 301 if (mode == sched_ctx->mode) 302 return 0; 303 304 switch (mode) { 305 case CDEV_SCHED_MODE_ROUNDROBIN: 306 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 307 roundrobin_scheduler) < 0) { 308 CS_LOG_ERR("Failed to load scheduler"); 309 return -1; 310 } 311 break; 312 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 313 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 314 pkt_size_based_distr_scheduler) < 0) { 315 CS_LOG_ERR("Failed to load scheduler"); 316 return -1; 317 } 318 break; 319 case CDEV_SCHED_MODE_FAILOVER: 320 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 321 failover_scheduler) < 0) { 322 CS_LOG_ERR("Failed to load scheduler"); 323 return -1; 324 } 325 break; 326 case CDEV_SCHED_MODE_MULTICORE: 327 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 328 multicore_scheduler) < 0) { 329 CS_LOG_ERR("Failed to load scheduler"); 330 return -1; 331 } 332 break; 333 default: 334 CS_LOG_ERR("Not yet supported"); 335 return -ENOTSUP; 336 } 337 338 return 0; 339 } 340 341 enum rte_cryptodev_scheduler_mode 342 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) 343 { 344 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 345 struct scheduler_ctx *sched_ctx; 346 347 if (!dev) { 348 CS_LOG_ERR("Operation not supported"); 349 return -ENOTSUP; 350 } 351 352 if (dev->driver_id != cryptodev_driver_id) { 353 CS_LOG_ERR("Operation not supported"); 354 return -ENOTSUP; 355 } 356 357 sched_ctx = dev->data->dev_private; 358 359 return sched_ctx->mode; 360 } 361 362 int 363 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, 364 uint32_t enable_reorder) 365 { 366 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 367 struct scheduler_ctx *sched_ctx; 368 369 if (!dev) { 370 CS_LOG_ERR("Operation not supported"); 371 return -ENOTSUP; 372 } 373 374 if (dev->driver_id != cryptodev_driver_id) { 375 CS_LOG_ERR("Operation not supported"); 376 return -ENOTSUP; 377 } 378 379 if (dev->data->dev_started) { 380 CS_LOG_ERR("Illegal operation"); 381 return -EBUSY; 382 } 383 384 sched_ctx = dev->data->dev_private; 385 386 sched_ctx->reordering_enabled = enable_reorder; 387 388 return 0; 389 } 390 391 int 392 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) 393 { 394 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 395 struct scheduler_ctx *sched_ctx; 396 397 if (!dev) { 398 CS_LOG_ERR("Operation not supported"); 399 return -ENOTSUP; 400 } 401 402 if (dev->driver_id != cryptodev_driver_id) { 403 CS_LOG_ERR("Operation not supported"); 404 return -ENOTSUP; 405 } 406 407 sched_ctx = dev->data->dev_private; 408 409 return (int)sched_ctx->reordering_enabled; 410 } 411 412 int 413 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, 414 struct rte_cryptodev_scheduler *scheduler) { 415 416 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 417 struct scheduler_ctx *sched_ctx; 418 419 if (!dev) { 420 CS_LOG_ERR("Operation not supported"); 421 return -ENOTSUP; 422 } 423 424 if (dev->driver_id != cryptodev_driver_id) { 425 CS_LOG_ERR("Operation not supported"); 426 return -ENOTSUP; 427 } 428 429 if (dev->data->dev_started) { 430 CS_LOG_ERR("Illegal operation"); 431 return -EBUSY; 432 } 433 434 sched_ctx = dev->data->dev_private; 435 436 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 437 CS_LOG_ERR("Invalid name %s, should be less than " 438 "%u bytes.\n", scheduler->name, 439 RTE_CRYPTODEV_NAME_MAX_LEN); 440 return -EINVAL; 441 } 442 strncpy(sched_ctx->name, scheduler->name, 443 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); 444 445 if (strlen(scheduler->description) > 446 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { 447 CS_LOG_ERR("Invalid description %s, should be less than " 448 "%u bytes.\n", scheduler->description, 449 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); 450 return -EINVAL; 451 } 452 strncpy(sched_ctx->description, scheduler->description, 453 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN); 454 455 /* load scheduler instance operations functions */ 456 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair; 457 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx; 458 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start; 459 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop; 460 sched_ctx->ops.slave_attach = scheduler->ops->slave_attach; 461 sched_ctx->ops.slave_detach = scheduler->ops->slave_detach; 462 sched_ctx->ops.option_set = scheduler->ops->option_set; 463 sched_ctx->ops.option_get = scheduler->ops->option_get; 464 465 if (sched_ctx->private_ctx) 466 rte_free(sched_ctx->private_ctx); 467 468 if (sched_ctx->ops.create_private_ctx) { 469 int ret = (*sched_ctx->ops.create_private_ctx)(dev); 470 471 if (ret < 0) { 472 CS_LOG_ERR("Unable to create scheduler private " 473 "context"); 474 return ret; 475 } 476 } 477 478 sched_ctx->mode = scheduler->mode; 479 480 return 0; 481 } 482 483 int 484 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves) 485 { 486 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 487 struct scheduler_ctx *sched_ctx; 488 uint32_t nb_slaves = 0; 489 490 if (!dev) { 491 CS_LOG_ERR("Operation not supported"); 492 return -ENOTSUP; 493 } 494 495 if (dev->driver_id != cryptodev_driver_id) { 496 CS_LOG_ERR("Operation not supported"); 497 return -ENOTSUP; 498 } 499 500 sched_ctx = dev->data->dev_private; 501 502 nb_slaves = sched_ctx->nb_slaves; 503 504 if (slaves && nb_slaves) { 505 uint32_t i; 506 507 for (i = 0; i < nb_slaves; i++) 508 slaves[i] = sched_ctx->slaves[i].dev_id; 509 } 510 511 return (int)nb_slaves; 512 } 513 514 int 515 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id, 516 enum rte_cryptodev_schedule_option_type option_type, 517 void *option) 518 { 519 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 520 struct scheduler_ctx *sched_ctx; 521 522 if (option_type == CDEV_SCHED_OPTION_NOT_SET || 523 option_type >= CDEV_SCHED_OPTION_COUNT) { 524 CS_LOG_ERR("Invalid option parameter"); 525 return -EINVAL; 526 } 527 528 if (!option) { 529 CS_LOG_ERR("Invalid option parameter"); 530 return -EINVAL; 531 } 532 533 if (dev->data->dev_started) { 534 CS_LOG_ERR("Illegal operation"); 535 return -EBUSY; 536 } 537 538 sched_ctx = dev->data->dev_private; 539 540 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP); 541 542 return (*sched_ctx->ops.option_set)(dev, option_type, option); 543 } 544 545 int 546 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, 547 enum rte_cryptodev_schedule_option_type option_type, 548 void *option) 549 { 550 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 551 struct scheduler_ctx *sched_ctx; 552 553 if (!dev) { 554 CS_LOG_ERR("Operation not supported"); 555 return -ENOTSUP; 556 } 557 558 if (!option) { 559 CS_LOG_ERR("Invalid option parameter"); 560 return -EINVAL; 561 } 562 563 if (dev->driver_id != cryptodev_driver_id) { 564 CS_LOG_ERR("Operation not supported"); 565 return -ENOTSUP; 566 } 567 568 sched_ctx = dev->data->dev_private; 569 570 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP); 571 572 return (*sched_ctx->ops.option_get)(dev, option_type, option); 573 } 574