1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_cryptodev.h> 34 #include <rte_malloc.h> 35 36 #include "rte_cryptodev_scheduler_operations.h" 37 #include "scheduler_pmd_private.h" 38 39 struct rr_scheduler_qp_ctx { 40 struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; 41 uint32_t nb_slaves; 42 43 uint32_t last_enq_slave_idx; 44 uint32_t last_deq_slave_idx; 45 }; 46 47 static uint16_t 48 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 49 { 50 struct rr_scheduler_qp_ctx *rr_qp_ctx = 51 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 52 uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; 53 struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; 54 uint16_t i, processed_ops; 55 struct rte_cryptodev_sym_session *sessions[nb_ops]; 56 struct scheduler_session *sess0, *sess1, *sess2, *sess3; 57 58 if (unlikely(nb_ops == 0)) 59 return 0; 60 61 for (i = 0; i < nb_ops && i < 4; i++) 62 rte_prefetch0(ops[i]->sym->session); 63 64 for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { 65 sess0 = (struct scheduler_session *) 66 ops[i]->sym->session->_private; 67 sess1 = (struct scheduler_session *) 68 ops[i+1]->sym->session->_private; 69 sess2 = (struct scheduler_session *) 70 ops[i+2]->sym->session->_private; 71 sess3 = (struct scheduler_session *) 72 ops[i+3]->sym->session->_private; 73 74 sessions[i] = ops[i]->sym->session; 75 sessions[i + 1] = ops[i + 1]->sym->session; 76 sessions[i + 2] = ops[i + 2]->sym->session; 77 sessions[i + 3] = ops[i + 3]->sym->session; 78 79 ops[i]->sym->session = sess0->sessions[slave_idx]; 80 ops[i + 1]->sym->session = sess1->sessions[slave_idx]; 81 ops[i + 2]->sym->session = sess2->sessions[slave_idx]; 82 ops[i + 3]->sym->session = sess3->sessions[slave_idx]; 83 84 rte_prefetch0(ops[i + 4]->sym->session); 85 rte_prefetch0(ops[i + 5]->sym->session); 86 rte_prefetch0(ops[i + 6]->sym->session); 87 rte_prefetch0(ops[i + 7]->sym->session); 88 } 89 90 for (; i < nb_ops; i++) { 91 sess0 = (struct scheduler_session *) 92 ops[i]->sym->session->_private; 93 sessions[i] = ops[i]->sym->session; 94 ops[i]->sym->session = sess0->sessions[slave_idx]; 95 } 96 97 processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, 98 slave->qp_id, ops, nb_ops); 99 100 slave->nb_inflight_cops += processed_ops; 101 102 rr_qp_ctx->last_enq_slave_idx += 1; 103 rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; 104 105 /* recover session if enqueue is failed */ 106 if (unlikely(processed_ops < nb_ops)) { 107 for (i = processed_ops; i < nb_ops; i++) 108 ops[i]->sym->session = sessions[i]; 109 } 110 111 return processed_ops; 112 } 113 114 static uint16_t 115 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, 116 uint16_t nb_ops) 117 { 118 struct rte_ring *order_ring = 119 ((struct scheduler_qp_ctx *)qp)->order_ring; 120 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, 121 nb_ops); 122 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, 123 nb_ops_to_enq); 124 125 scheduler_order_insert(order_ring, ops, nb_ops_enqd); 126 127 return nb_ops_enqd; 128 } 129 130 131 static uint16_t 132 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 133 { 134 struct rr_scheduler_qp_ctx *rr_qp_ctx = 135 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 136 struct scheduler_slave *slave; 137 uint32_t last_slave_idx = rr_qp_ctx->last_deq_slave_idx; 138 uint16_t nb_deq_ops; 139 140 if (unlikely(rr_qp_ctx->slaves[last_slave_idx].nb_inflight_cops == 0)) { 141 do { 142 last_slave_idx += 1; 143 144 if (unlikely(last_slave_idx >= rr_qp_ctx->nb_slaves)) 145 last_slave_idx = 0; 146 /* looped back, means no inflight cops in the queue */ 147 if (last_slave_idx == rr_qp_ctx->last_deq_slave_idx) 148 return 0; 149 } while (rr_qp_ctx->slaves[last_slave_idx].nb_inflight_cops 150 == 0); 151 } 152 153 slave = &rr_qp_ctx->slaves[last_slave_idx]; 154 155 nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id, 156 slave->qp_id, ops, nb_ops); 157 158 last_slave_idx += 1; 159 last_slave_idx %= rr_qp_ctx->nb_slaves; 160 161 rr_qp_ctx->last_deq_slave_idx = last_slave_idx; 162 163 slave->nb_inflight_cops -= nb_deq_ops; 164 165 return nb_deq_ops; 166 } 167 168 static uint16_t 169 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, 170 uint16_t nb_ops) 171 { 172 struct rte_ring *order_ring = 173 ((struct scheduler_qp_ctx *)qp)->order_ring; 174 175 schedule_dequeue(qp, ops, nb_ops); 176 177 return scheduler_order_drain(order_ring, ops, nb_ops); 178 } 179 180 static int 181 slave_attach(__rte_unused struct rte_cryptodev *dev, 182 __rte_unused uint8_t slave_id) 183 { 184 return 0; 185 } 186 187 static int 188 slave_detach(__rte_unused struct rte_cryptodev *dev, 189 __rte_unused uint8_t slave_id) 190 { 191 return 0; 192 } 193 194 static int 195 scheduler_start(struct rte_cryptodev *dev) 196 { 197 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 198 uint16_t i; 199 200 if (sched_ctx->reordering_enabled) { 201 dev->enqueue_burst = &schedule_enqueue_ordering; 202 dev->dequeue_burst = &schedule_dequeue_ordering; 203 } else { 204 dev->enqueue_burst = &schedule_enqueue; 205 dev->dequeue_burst = &schedule_dequeue; 206 } 207 208 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 209 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; 210 struct rr_scheduler_qp_ctx *rr_qp_ctx = 211 qp_ctx->private_qp_ctx; 212 uint32_t j; 213 214 memset(rr_qp_ctx->slaves, 0, 215 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES * 216 sizeof(struct scheduler_slave)); 217 for (j = 0; j < sched_ctx->nb_slaves; j++) { 218 rr_qp_ctx->slaves[j].dev_id = 219 sched_ctx->slaves[j].dev_id; 220 rr_qp_ctx->slaves[j].qp_id = i; 221 } 222 223 rr_qp_ctx->nb_slaves = sched_ctx->nb_slaves; 224 225 rr_qp_ctx->last_enq_slave_idx = 0; 226 rr_qp_ctx->last_deq_slave_idx = 0; 227 } 228 229 return 0; 230 } 231 232 static int 233 scheduler_stop(__rte_unused struct rte_cryptodev *dev) 234 { 235 return 0; 236 } 237 238 static int 239 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) 240 { 241 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; 242 struct rr_scheduler_qp_ctx *rr_qp_ctx; 243 244 rr_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*rr_qp_ctx), 0, 245 rte_socket_id()); 246 if (!rr_qp_ctx) { 247 CS_LOG_ERR("failed allocate memory for private queue pair"); 248 return -ENOMEM; 249 } 250 251 qp_ctx->private_qp_ctx = (void *)rr_qp_ctx; 252 253 return 0; 254 } 255 256 static int 257 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) 258 { 259 return 0; 260 } 261 262 struct rte_cryptodev_scheduler_ops scheduler_rr_ops = { 263 slave_attach, 264 slave_detach, 265 scheduler_start, 266 scheduler_stop, 267 scheduler_config_qp, 268 scheduler_create_private_ctx, 269 NULL, /* option_set */ 270 NULL /* option_get */ 271 }; 272 273 struct rte_cryptodev_scheduler scheduler = { 274 .name = "roundrobin-scheduler", 275 .description = "scheduler which will round robin burst across " 276 "slave crypto devices", 277 .mode = CDEV_SCHED_MODE_ROUNDROBIN, 278 .ops = &scheduler_rr_ops 279 }; 280 281 struct rte_cryptodev_scheduler *roundrobin_scheduler = &scheduler; 282