1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <rte_cryptodev.h> 6 #include <rte_malloc.h> 7 8 #include "rte_cryptodev_scheduler_operations.h" 9 #include "scheduler_pmd_private.h" 10 11 struct rr_scheduler_qp_ctx { 12 struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; 13 uint32_t nb_slaves; 14 15 uint32_t last_enq_slave_idx; 16 uint32_t last_deq_slave_idx; 17 }; 18 19 static uint16_t 20 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 21 { 22 struct rr_scheduler_qp_ctx *rr_qp_ctx = 23 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 24 uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; 25 struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; 26 uint16_t i, processed_ops; 27 28 if (unlikely(nb_ops == 0)) 29 return 0; 30 31 for (i = 0; i < nb_ops && i < 4; i++) 32 rte_prefetch0(ops[i]->sym->session); 33 34 processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, 35 slave->qp_id, ops, nb_ops); 36 37 slave->nb_inflight_cops += processed_ops; 38 39 rr_qp_ctx->last_enq_slave_idx += 1; 40 rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; 41 42 return processed_ops; 43 } 44 45 static uint16_t 46 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, 47 uint16_t nb_ops) 48 { 49 struct rte_ring *order_ring = 50 ((struct scheduler_qp_ctx *)qp)->order_ring; 51 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, 52 nb_ops); 53 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, 54 nb_ops_to_enq); 55 56 scheduler_order_insert(order_ring, ops, nb_ops_enqd); 57 58 return nb_ops_enqd; 59 } 60 61 62 static uint16_t 63 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 64 { 65 struct rr_scheduler_qp_ctx *rr_qp_ctx = 66 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 67 struct scheduler_slave *slave; 68 uint32_t last_slave_idx = rr_qp_ctx->last_deq_slave_idx; 69 uint16_t nb_deq_ops; 70 71 if (unlikely(rr_qp_ctx->slaves[last_slave_idx].nb_inflight_cops == 0)) { 72 do { 73 last_slave_idx += 1; 74 75 if (unlikely(last_slave_idx >= rr_qp_ctx->nb_slaves)) 76 last_slave_idx = 0; 77 /* looped back, means no inflight cops in the queue */ 78 if (last_slave_idx == rr_qp_ctx->last_deq_slave_idx) 79 return 0; 80 } while (rr_qp_ctx->slaves[last_slave_idx].nb_inflight_cops 81 == 0); 82 } 83 84 slave = &rr_qp_ctx->slaves[last_slave_idx]; 85 86 nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id, 87 slave->qp_id, ops, nb_ops); 88 89 last_slave_idx += 1; 90 last_slave_idx %= rr_qp_ctx->nb_slaves; 91 92 rr_qp_ctx->last_deq_slave_idx = last_slave_idx; 93 94 slave->nb_inflight_cops -= nb_deq_ops; 95 96 return nb_deq_ops; 97 } 98 99 static uint16_t 100 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, 101 uint16_t nb_ops) 102 { 103 struct rte_ring *order_ring = 104 ((struct scheduler_qp_ctx *)qp)->order_ring; 105 106 schedule_dequeue(qp, ops, nb_ops); 107 108 return scheduler_order_drain(order_ring, ops, nb_ops); 109 } 110 111 static int 112 slave_attach(__rte_unused struct rte_cryptodev *dev, 113 __rte_unused uint8_t slave_id) 114 { 115 return 0; 116 } 117 118 static int 119 slave_detach(__rte_unused struct rte_cryptodev *dev, 120 __rte_unused uint8_t slave_id) 121 { 122 return 0; 123 } 124 125 static int 126 scheduler_start(struct rte_cryptodev *dev) 127 { 128 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 129 uint16_t i; 130 131 if (sched_ctx->reordering_enabled) { 132 dev->enqueue_burst = &schedule_enqueue_ordering; 133 dev->dequeue_burst = &schedule_dequeue_ordering; 134 } else { 135 dev->enqueue_burst = &schedule_enqueue; 136 dev->dequeue_burst = &schedule_dequeue; 137 } 138 139 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 140 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; 141 struct rr_scheduler_qp_ctx *rr_qp_ctx = 142 qp_ctx->private_qp_ctx; 143 uint32_t j; 144 145 memset(rr_qp_ctx->slaves, 0, 146 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES * 147 sizeof(struct scheduler_slave)); 148 for (j = 0; j < sched_ctx->nb_slaves; j++) { 149 rr_qp_ctx->slaves[j].dev_id = 150 sched_ctx->slaves[j].dev_id; 151 rr_qp_ctx->slaves[j].qp_id = i; 152 } 153 154 rr_qp_ctx->nb_slaves = sched_ctx->nb_slaves; 155 156 rr_qp_ctx->last_enq_slave_idx = 0; 157 rr_qp_ctx->last_deq_slave_idx = 0; 158 } 159 160 return 0; 161 } 162 163 static int 164 scheduler_stop(__rte_unused struct rte_cryptodev *dev) 165 { 166 return 0; 167 } 168 169 static int 170 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) 171 { 172 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; 173 struct rr_scheduler_qp_ctx *rr_qp_ctx; 174 175 rr_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*rr_qp_ctx), 0, 176 rte_socket_id()); 177 if (!rr_qp_ctx) { 178 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair"); 179 return -ENOMEM; 180 } 181 182 qp_ctx->private_qp_ctx = (void *)rr_qp_ctx; 183 184 return 0; 185 } 186 187 static int 188 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) 189 { 190 return 0; 191 } 192 193 static struct rte_cryptodev_scheduler_ops scheduler_rr_ops = { 194 slave_attach, 195 slave_detach, 196 scheduler_start, 197 scheduler_stop, 198 scheduler_config_qp, 199 scheduler_create_private_ctx, 200 NULL, /* option_set */ 201 NULL /* option_get */ 202 }; 203 204 static struct rte_cryptodev_scheduler scheduler = { 205 .name = "roundrobin-scheduler", 206 .description = "scheduler which will round robin burst across " 207 "slave crypto devices", 208 .mode = CDEV_SCHED_MODE_ROUNDROBIN, 209 .ops = &scheduler_rr_ops 210 }; 211 212 struct rte_cryptodev_scheduler *crypto_scheduler_roundrobin = &scheduler; 213