1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <rte_cryptodev.h> 6 #include <rte_malloc.h> 7 8 #include "rte_cryptodev_scheduler_operations.h" 9 #include "scheduler_pmd_private.h" 10 11 #define DEF_PKT_SIZE_THRESHOLD (0xffffff80) 12 #define WORKER_IDX_SWITCH_MASK (0x01) 13 #define PRIMARY_WORKER_IDX 0 14 #define SECONDARY_WORKER_IDX 1 15 #define NB_PKT_SIZE_WORKERS 2 16 17 /** pkt size based scheduler context */ 18 struct psd_scheduler_ctx { 19 uint32_t threshold; 20 }; 21 22 /** pkt size based scheduler queue pair context */ 23 struct psd_scheduler_qp_ctx { 24 struct scheduler_worker primary_worker; 25 struct scheduler_worker secondary_worker; 26 uint32_t threshold; 27 uint8_t deq_idx; 28 } __rte_cache_aligned; 29 30 /** scheduling operation variables' wrapping */ 31 struct psd_schedule_op { 32 uint8_t worker_idx; 33 uint16_t pos; 34 }; 35 36 static uint16_t 37 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 38 { 39 struct scheduler_qp_ctx *qp_ctx = qp; 40 struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx; 41 struct rte_crypto_op *sched_ops[NB_PKT_SIZE_WORKERS][nb_ops]; 42 uint32_t in_flight_ops[NB_PKT_SIZE_WORKERS] = { 43 psd_qp_ctx->primary_worker.nb_inflight_cops, 44 psd_qp_ctx->secondary_worker.nb_inflight_cops 45 }; 46 struct psd_schedule_op enq_ops[NB_PKT_SIZE_WORKERS] = { 47 {PRIMARY_WORKER_IDX, 0}, {SECONDARY_WORKER_IDX, 0} 48 }; 49 struct psd_schedule_op *p_enq_op; 50 uint16_t i, processed_ops_pri = 0, processed_ops_sec = 0; 51 uint32_t job_len; 52 53 if (unlikely(nb_ops == 0)) 54 return 0; 55 56 for (i = 0; i < nb_ops && i < 4; i++) { 57 rte_prefetch0(ops[i]->sym); 58 rte_prefetch0(ops[i]->sym->session); 59 } 60 61 for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { 62 rte_prefetch0(ops[i + 4]->sym); 63 rte_prefetch0(ops[i + 4]->sym->session); 64 rte_prefetch0(ops[i + 5]->sym); 65 rte_prefetch0(ops[i + 5]->sym->session); 66 rte_prefetch0(ops[i + 6]->sym); 67 rte_prefetch0(ops[i + 6]->sym->session); 68 rte_prefetch0(ops[i + 7]->sym); 69 rte_prefetch0(ops[i + 7]->sym->session); 70 71 /* job_len is initialized as cipher data length, once 72 * it is 0, equals to auth data length 73 */ 74 job_len = ops[i]->sym->cipher.data.length; 75 job_len += (ops[i]->sym->cipher.data.length == 0) * 76 ops[i]->sym->auth.data.length; 77 /* decide the target op based on the job length */ 78 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; 79 80 /* stop schedule cops before the queue is full, this shall 81 * prevent the failed enqueue 82 */ 83 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] == 84 qp_ctx->max_nb_objs) { 85 i = nb_ops; 86 break; 87 } 88 89 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i]; 90 p_enq_op->pos++; 91 92 job_len = ops[i+1]->sym->cipher.data.length; 93 job_len += (ops[i+1]->sym->cipher.data.length == 0) * 94 ops[i+1]->sym->auth.data.length; 95 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; 96 97 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] == 98 qp_ctx->max_nb_objs) { 99 i = nb_ops; 100 break; 101 } 102 103 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+1]; 104 p_enq_op->pos++; 105 106 job_len = ops[i+2]->sym->cipher.data.length; 107 job_len += (ops[i+2]->sym->cipher.data.length == 0) * 108 ops[i+2]->sym->auth.data.length; 109 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; 110 111 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] == 112 qp_ctx->max_nb_objs) { 113 i = nb_ops; 114 break; 115 } 116 117 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+2]; 118 p_enq_op->pos++; 119 120 job_len = ops[i+3]->sym->cipher.data.length; 121 job_len += (ops[i+3]->sym->cipher.data.length == 0) * 122 ops[i+3]->sym->auth.data.length; 123 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; 124 125 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] == 126 qp_ctx->max_nb_objs) { 127 i = nb_ops; 128 break; 129 } 130 131 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+3]; 132 p_enq_op->pos++; 133 } 134 135 for (; i < nb_ops; i++) { 136 job_len = ops[i]->sym->cipher.data.length; 137 job_len += (ops[i]->sym->cipher.data.length == 0) * 138 ops[i]->sym->auth.data.length; 139 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; 140 141 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] == 142 qp_ctx->max_nb_objs) { 143 i = nb_ops; 144 break; 145 } 146 147 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i]; 148 p_enq_op->pos++; 149 } 150 151 processed_ops_pri = rte_cryptodev_enqueue_burst( 152 psd_qp_ctx->primary_worker.dev_id, 153 psd_qp_ctx->primary_worker.qp_id, 154 sched_ops[PRIMARY_WORKER_IDX], 155 enq_ops[PRIMARY_WORKER_IDX].pos); 156 /* enqueue shall not fail as the worker queue is monitored */ 157 RTE_ASSERT(processed_ops_pri == enq_ops[PRIMARY_WORKER_IDX].pos); 158 159 psd_qp_ctx->primary_worker.nb_inflight_cops += processed_ops_pri; 160 161 processed_ops_sec = rte_cryptodev_enqueue_burst( 162 psd_qp_ctx->secondary_worker.dev_id, 163 psd_qp_ctx->secondary_worker.qp_id, 164 sched_ops[SECONDARY_WORKER_IDX], 165 enq_ops[SECONDARY_WORKER_IDX].pos); 166 RTE_ASSERT(processed_ops_sec == enq_ops[SECONDARY_WORKER_IDX].pos); 167 168 psd_qp_ctx->secondary_worker.nb_inflight_cops += processed_ops_sec; 169 170 return processed_ops_pri + processed_ops_sec; 171 } 172 173 static uint16_t 174 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, 175 uint16_t nb_ops) 176 { 177 struct rte_ring *order_ring = 178 ((struct scheduler_qp_ctx *)qp)->order_ring; 179 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, 180 nb_ops); 181 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, 182 nb_ops_to_enq); 183 184 scheduler_order_insert(order_ring, ops, nb_ops_enqd); 185 186 return nb_ops_enqd; 187 } 188 189 static uint16_t 190 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 191 { 192 struct psd_scheduler_qp_ctx *qp_ctx = 193 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 194 struct scheduler_worker *workers[NB_PKT_SIZE_WORKERS] = { 195 &qp_ctx->primary_worker, &qp_ctx->secondary_worker}; 196 struct scheduler_worker *worker = workers[qp_ctx->deq_idx]; 197 uint16_t nb_deq_ops_pri = 0, nb_deq_ops_sec = 0; 198 199 if (worker->nb_inflight_cops) { 200 nb_deq_ops_pri = rte_cryptodev_dequeue_burst(worker->dev_id, 201 worker->qp_id, ops, nb_ops); 202 worker->nb_inflight_cops -= nb_deq_ops_pri; 203 } 204 205 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_IDX_SWITCH_MASK; 206 207 if (nb_deq_ops_pri == nb_ops) 208 return nb_deq_ops_pri; 209 210 worker = workers[qp_ctx->deq_idx]; 211 212 if (worker->nb_inflight_cops) { 213 nb_deq_ops_sec = rte_cryptodev_dequeue_burst(worker->dev_id, 214 worker->qp_id, &ops[nb_deq_ops_pri], 215 nb_ops - nb_deq_ops_pri); 216 worker->nb_inflight_cops -= nb_deq_ops_sec; 217 218 if (!worker->nb_inflight_cops) 219 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & 220 WORKER_IDX_SWITCH_MASK; 221 } 222 223 return nb_deq_ops_pri + nb_deq_ops_sec; 224 } 225 226 static uint16_t 227 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, 228 uint16_t nb_ops) 229 { 230 struct rte_ring *order_ring = 231 ((struct scheduler_qp_ctx *)qp)->order_ring; 232 233 schedule_dequeue(qp, ops, nb_ops); 234 235 return scheduler_order_drain(order_ring, ops, nb_ops); 236 } 237 238 static int 239 worker_attach(__rte_unused struct rte_cryptodev *dev, 240 __rte_unused uint8_t worker_id) 241 { 242 return 0; 243 } 244 245 static int 246 worker_detach(__rte_unused struct rte_cryptodev *dev, 247 __rte_unused uint8_t worker_id) 248 { 249 return 0; 250 } 251 252 static int 253 scheduler_start(struct rte_cryptodev *dev) 254 { 255 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 256 struct psd_scheduler_ctx *psd_ctx = sched_ctx->private_ctx; 257 uint16_t i; 258 259 /* for packet size based scheduler, nb_workers have to >= 2 */ 260 if (sched_ctx->nb_workers < NB_PKT_SIZE_WORKERS) { 261 CR_SCHED_LOG(ERR, "not enough workers to start"); 262 return -1; 263 } 264 265 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 266 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; 267 struct psd_scheduler_qp_ctx *ps_qp_ctx = 268 qp_ctx->private_qp_ctx; 269 270 ps_qp_ctx->primary_worker.dev_id = 271 sched_ctx->workers[PRIMARY_WORKER_IDX].dev_id; 272 ps_qp_ctx->primary_worker.qp_id = i; 273 ps_qp_ctx->primary_worker.nb_inflight_cops = 0; 274 275 ps_qp_ctx->secondary_worker.dev_id = 276 sched_ctx->workers[SECONDARY_WORKER_IDX].dev_id; 277 ps_qp_ctx->secondary_worker.qp_id = i; 278 ps_qp_ctx->secondary_worker.nb_inflight_cops = 0; 279 280 ps_qp_ctx->threshold = psd_ctx->threshold; 281 } 282 283 if (sched_ctx->reordering_enabled) { 284 dev->enqueue_burst = &schedule_enqueue_ordering; 285 dev->dequeue_burst = &schedule_dequeue_ordering; 286 } else { 287 dev->enqueue_burst = &schedule_enqueue; 288 dev->dequeue_burst = &schedule_dequeue; 289 } 290 291 return 0; 292 } 293 294 static int 295 scheduler_stop(struct rte_cryptodev *dev) 296 { 297 uint16_t i; 298 299 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 300 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; 301 struct psd_scheduler_qp_ctx *ps_qp_ctx = qp_ctx->private_qp_ctx; 302 303 if (ps_qp_ctx->primary_worker.nb_inflight_cops + 304 ps_qp_ctx->secondary_worker.nb_inflight_cops) { 305 CR_SCHED_LOG(ERR, "Some crypto ops left in worker queue"); 306 return -1; 307 } 308 } 309 310 return 0; 311 } 312 313 static int 314 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) 315 { 316 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; 317 struct psd_scheduler_qp_ctx *ps_qp_ctx; 318 319 ps_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*ps_qp_ctx), 0, 320 rte_socket_id()); 321 if (!ps_qp_ctx) { 322 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair"); 323 return -ENOMEM; 324 } 325 326 qp_ctx->private_qp_ctx = (void *)ps_qp_ctx; 327 328 return 0; 329 } 330 331 static int 332 scheduler_create_private_ctx(struct rte_cryptodev *dev) 333 { 334 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 335 struct psd_scheduler_ctx *psd_ctx; 336 337 if (sched_ctx->private_ctx) { 338 rte_free(sched_ctx->private_ctx); 339 sched_ctx->private_ctx = NULL; 340 } 341 342 psd_ctx = rte_zmalloc_socket(NULL, sizeof(struct psd_scheduler_ctx), 0, 343 rte_socket_id()); 344 if (!psd_ctx) { 345 CR_SCHED_LOG(ERR, "failed allocate memory"); 346 return -ENOMEM; 347 } 348 349 psd_ctx->threshold = DEF_PKT_SIZE_THRESHOLD; 350 351 sched_ctx->private_ctx = (void *)psd_ctx; 352 353 return 0; 354 } 355 static int 356 scheduler_option_set(struct rte_cryptodev *dev, uint32_t option_type, 357 void *option) 358 { 359 struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *) 360 dev->data->dev_private)->private_ctx; 361 uint32_t threshold; 362 363 if ((enum rte_cryptodev_schedule_option_type)option_type != 364 CDEV_SCHED_OPTION_THRESHOLD) { 365 CR_SCHED_LOG(ERR, "Option not supported"); 366 return -EINVAL; 367 } 368 369 threshold = ((struct rte_cryptodev_scheduler_threshold_option *) 370 option)->threshold; 371 if (!rte_is_power_of_2(threshold)) { 372 CR_SCHED_LOG(ERR, "Threshold is not power of 2"); 373 return -EINVAL; 374 } 375 376 psd_ctx->threshold = ~(threshold - 1); 377 378 return 0; 379 } 380 381 static int 382 scheduler_option_get(struct rte_cryptodev *dev, uint32_t option_type, 383 void *option) 384 { 385 struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *) 386 dev->data->dev_private)->private_ctx; 387 struct rte_cryptodev_scheduler_threshold_option *threshold_option; 388 389 if ((enum rte_cryptodev_schedule_option_type)option_type != 390 CDEV_SCHED_OPTION_THRESHOLD) { 391 CR_SCHED_LOG(ERR, "Option not supported"); 392 return -EINVAL; 393 } 394 395 threshold_option = option; 396 threshold_option->threshold = (~psd_ctx->threshold) + 1; 397 398 return 0; 399 } 400 401 static struct rte_cryptodev_scheduler_ops scheduler_ps_ops = { 402 worker_attach, 403 worker_detach, 404 scheduler_start, 405 scheduler_stop, 406 scheduler_config_qp, 407 scheduler_create_private_ctx, 408 scheduler_option_set, 409 scheduler_option_get 410 }; 411 412 static struct rte_cryptodev_scheduler psd_scheduler = { 413 .name = "packet-size-based-scheduler", 414 .description = "scheduler which will distribute crypto op " 415 "burst based on the packet size", 416 .mode = CDEV_SCHED_MODE_PKT_SIZE_DISTR, 417 .ops = &scheduler_ps_ops 418 }; 419 420 struct rte_cryptodev_scheduler *crypto_scheduler_pkt_size_based_distr = &psd_scheduler; 421