1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <cryptodev_pmd.h> 6 #include <rte_malloc.h> 7 8 #include "rte_cryptodev_scheduler_operations.h" 9 #include "scheduler_pmd_private.h" 10 11 struct rr_scheduler_qp_ctx { 12 struct scheduler_worker workers[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS]; 13 uint32_t nb_workers; 14 15 uint32_t last_enq_worker_idx; 16 uint32_t last_deq_worker_idx; 17 }; 18 19 static uint16_t 20 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 21 { 22 struct rr_scheduler_qp_ctx *rr_qp_ctx = 23 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 24 uint32_t worker_idx = rr_qp_ctx->last_enq_worker_idx; 25 struct scheduler_worker *worker = &rr_qp_ctx->workers[worker_idx]; 26 uint16_t i, processed_ops; 27 28 if (unlikely(nb_ops == 0)) 29 return 0; 30 31 for (i = 0; i < nb_ops && i < 4; i++) 32 rte_prefetch0(ops[i]->sym->session); 33 34 processed_ops = rte_cryptodev_enqueue_burst(worker->dev_id, 35 worker->qp_id, ops, nb_ops); 36 37 worker->nb_inflight_cops += processed_ops; 38 39 rr_qp_ctx->last_enq_worker_idx += 1; 40 rr_qp_ctx->last_enq_worker_idx %= rr_qp_ctx->nb_workers; 41 42 return processed_ops; 43 } 44 45 static uint16_t 46 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, 47 uint16_t nb_ops) 48 { 49 struct rte_ring *order_ring = 50 ((struct scheduler_qp_ctx *)qp)->order_ring; 51 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, 52 nb_ops); 53 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, 54 nb_ops_to_enq); 55 56 scheduler_order_insert(order_ring, ops, nb_ops_enqd); 57 58 return nb_ops_enqd; 59 } 60 61 62 static uint16_t 63 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 64 { 65 struct rr_scheduler_qp_ctx *rr_qp_ctx = 66 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 67 struct scheduler_worker *worker; 68 uint32_t last_worker_idx = rr_qp_ctx->last_deq_worker_idx; 69 uint16_t nb_deq_ops; 70 71 if (unlikely(rr_qp_ctx->workers[last_worker_idx].nb_inflight_cops 72 == 0)) { 73 do { 74 last_worker_idx += 1; 75 76 if (unlikely(last_worker_idx >= rr_qp_ctx->nb_workers)) 77 last_worker_idx = 0; 78 /* looped back, means no inflight cops in the queue */ 79 if (last_worker_idx == rr_qp_ctx->last_deq_worker_idx) 80 return 0; 81 } while (rr_qp_ctx->workers[last_worker_idx].nb_inflight_cops 82 == 0); 83 } 84 85 worker = &rr_qp_ctx->workers[last_worker_idx]; 86 87 nb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id, 88 worker->qp_id, ops, nb_ops); 89 90 last_worker_idx += 1; 91 last_worker_idx %= rr_qp_ctx->nb_workers; 92 93 rr_qp_ctx->last_deq_worker_idx = last_worker_idx; 94 95 worker->nb_inflight_cops -= nb_deq_ops; 96 97 return nb_deq_ops; 98 } 99 100 static uint16_t 101 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, 102 uint16_t nb_ops) 103 { 104 struct rte_ring *order_ring = 105 ((struct scheduler_qp_ctx *)qp)->order_ring; 106 107 schedule_dequeue(qp, ops, nb_ops); 108 109 return scheduler_order_drain(order_ring, ops, nb_ops); 110 } 111 112 static int 113 worker_attach(__rte_unused struct rte_cryptodev *dev, 114 __rte_unused uint8_t worker_id) 115 { 116 return 0; 117 } 118 119 static int 120 worker_detach(__rte_unused struct rte_cryptodev *dev, 121 __rte_unused uint8_t worker_id) 122 { 123 return 0; 124 } 125 126 static int 127 scheduler_start(struct rte_cryptodev *dev) 128 { 129 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 130 uint16_t i; 131 132 if (sched_ctx->reordering_enabled) { 133 dev->enqueue_burst = &schedule_enqueue_ordering; 134 dev->dequeue_burst = &schedule_dequeue_ordering; 135 } else { 136 dev->enqueue_burst = &schedule_enqueue; 137 dev->dequeue_burst = &schedule_dequeue; 138 } 139 140 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 141 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; 142 struct rr_scheduler_qp_ctx *rr_qp_ctx = 143 qp_ctx->private_qp_ctx; 144 uint32_t j; 145 146 memset(rr_qp_ctx->workers, 0, 147 RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS * 148 sizeof(struct scheduler_worker)); 149 for (j = 0; j < sched_ctx->nb_workers; j++) { 150 rr_qp_ctx->workers[j].dev_id = 151 sched_ctx->workers[j].dev_id; 152 rr_qp_ctx->workers[j].qp_id = i; 153 } 154 155 rr_qp_ctx->nb_workers = sched_ctx->nb_workers; 156 157 rr_qp_ctx->last_enq_worker_idx = 0; 158 rr_qp_ctx->last_deq_worker_idx = 0; 159 } 160 161 return 0; 162 } 163 164 static int 165 scheduler_stop(__rte_unused struct rte_cryptodev *dev) 166 { 167 return 0; 168 } 169 170 static int 171 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) 172 { 173 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; 174 struct rr_scheduler_qp_ctx *rr_qp_ctx; 175 176 rr_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*rr_qp_ctx), 0, 177 rte_socket_id()); 178 if (!rr_qp_ctx) { 179 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair"); 180 return -ENOMEM; 181 } 182 183 qp_ctx->private_qp_ctx = (void *)rr_qp_ctx; 184 185 return 0; 186 } 187 188 static int 189 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) 190 { 191 return 0; 192 } 193 194 static struct rte_cryptodev_scheduler_ops scheduler_rr_ops = { 195 worker_attach, 196 worker_detach, 197 scheduler_start, 198 scheduler_stop, 199 scheduler_config_qp, 200 scheduler_create_private_ctx, 201 NULL, /* option_set */ 202 NULL /* option_get */ 203 }; 204 205 static struct rte_cryptodev_scheduler scheduler = { 206 .name = "roundrobin-scheduler", 207 .description = "scheduler which will round robin burst across " 208 "worker crypto devices", 209 .mode = CDEV_SCHED_MODE_ROUNDROBIN, 210 .ops = &scheduler_rr_ops 211 }; 212 213 struct rte_cryptodev_scheduler *crypto_scheduler_roundrobin = &scheduler; 214