1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <cryptodev_pmd.h> 6 #include <rte_malloc.h> 7 8 #include "rte_cryptodev_scheduler_operations.h" 9 #include "scheduler_pmd_private.h" 10 11 #define PRIMARY_WORKER_IDX 0 12 #define SECONDARY_WORKER_IDX 1 13 #define NB_FAILOVER_WORKERS 2 14 #define WORKER_SWITCH_MASK (0x01) 15 16 struct fo_scheduler_qp_ctx { 17 struct scheduler_worker primary_worker; 18 struct scheduler_worker secondary_worker; 19 20 uint8_t deq_idx; 21 }; 22 23 static __rte_always_inline uint16_t 24 failover_worker_enqueue(struct scheduler_worker *worker, 25 struct rte_crypto_op **ops, uint16_t nb_ops) 26 { 27 uint16_t i, processed_ops; 28 29 for (i = 0; i < nb_ops && i < 4; i++) 30 rte_prefetch0(ops[i]->sym->session); 31 32 processed_ops = rte_cryptodev_enqueue_burst(worker->dev_id, 33 worker->qp_id, ops, nb_ops); 34 worker->nb_inflight_cops += processed_ops; 35 36 return processed_ops; 37 } 38 39 static uint16_t 40 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 41 { 42 struct fo_scheduler_qp_ctx *qp_ctx = 43 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 44 uint16_t enqueued_ops; 45 46 if (unlikely(nb_ops == 0)) 47 return 0; 48 49 enqueued_ops = failover_worker_enqueue(&qp_ctx->primary_worker, 50 ops, nb_ops); 51 52 if (enqueued_ops < nb_ops) 53 enqueued_ops += failover_worker_enqueue( 54 &qp_ctx->secondary_worker, 55 &ops[enqueued_ops], 56 nb_ops - enqueued_ops); 57 58 return enqueued_ops; 59 } 60 61 62 static uint16_t 63 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, 64 uint16_t nb_ops) 65 { 66 struct rte_ring *order_ring = 67 ((struct scheduler_qp_ctx *)qp)->order_ring; 68 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, 69 nb_ops); 70 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, 71 nb_ops_to_enq); 72 73 scheduler_order_insert(order_ring, ops, nb_ops_enqd); 74 75 return nb_ops_enqd; 76 } 77 78 static uint16_t 79 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 80 { 81 struct fo_scheduler_qp_ctx *qp_ctx = 82 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 83 struct scheduler_worker *workers[NB_FAILOVER_WORKERS] = { 84 &qp_ctx->primary_worker, &qp_ctx->secondary_worker}; 85 struct scheduler_worker *worker = workers[qp_ctx->deq_idx]; 86 uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0; 87 88 if (worker->nb_inflight_cops) { 89 nb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id, 90 worker->qp_id, ops, nb_ops); 91 worker->nb_inflight_cops -= nb_deq_ops; 92 } 93 94 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_SWITCH_MASK; 95 96 if (nb_deq_ops == nb_ops) 97 return nb_deq_ops; 98 99 worker = workers[qp_ctx->deq_idx]; 100 101 if (worker->nb_inflight_cops) { 102 nb_deq_ops2 = rte_cryptodev_dequeue_burst(worker->dev_id, 103 worker->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops); 104 worker->nb_inflight_cops -= nb_deq_ops2; 105 } 106 107 return nb_deq_ops + nb_deq_ops2; 108 } 109 110 static uint16_t 111 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, 112 uint16_t nb_ops) 113 { 114 struct rte_ring *order_ring = 115 ((struct scheduler_qp_ctx *)qp)->order_ring; 116 117 schedule_dequeue(qp, ops, nb_ops); 118 119 return scheduler_order_drain(order_ring, ops, nb_ops); 120 } 121 122 static int 123 worker_attach(__rte_unused struct rte_cryptodev *dev, 124 __rte_unused uint8_t worker_id) 125 { 126 return 0; 127 } 128 129 static int 130 worker_detach(__rte_unused struct rte_cryptodev *dev, 131 __rte_unused uint8_t worker_id) 132 { 133 return 0; 134 } 135 136 static int 137 scheduler_start(struct rte_cryptodev *dev) 138 { 139 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 140 uint16_t i; 141 142 if (sched_ctx->nb_workers < 2) { 143 CR_SCHED_LOG(ERR, "Number of workers shall no less than 2"); 144 return -ENOMEM; 145 } 146 147 if (sched_ctx->reordering_enabled) { 148 dev->enqueue_burst = schedule_enqueue_ordering; 149 dev->dequeue_burst = schedule_dequeue_ordering; 150 } else { 151 dev->enqueue_burst = schedule_enqueue; 152 dev->dequeue_burst = schedule_dequeue; 153 } 154 155 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 156 struct fo_scheduler_qp_ctx *qp_ctx = 157 ((struct scheduler_qp_ctx *) 158 dev->data->queue_pairs[i])->private_qp_ctx; 159 160 rte_memcpy(&qp_ctx->primary_worker, 161 &sched_ctx->workers[PRIMARY_WORKER_IDX], 162 sizeof(struct scheduler_worker)); 163 rte_memcpy(&qp_ctx->secondary_worker, 164 &sched_ctx->workers[SECONDARY_WORKER_IDX], 165 sizeof(struct scheduler_worker)); 166 } 167 168 return 0; 169 } 170 171 static int 172 scheduler_stop(__rte_unused struct rte_cryptodev *dev) 173 { 174 return 0; 175 } 176 177 static int 178 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) 179 { 180 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; 181 struct fo_scheduler_qp_ctx *fo_qp_ctx; 182 183 fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0, 184 rte_socket_id()); 185 if (!fo_qp_ctx) { 186 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair"); 187 return -ENOMEM; 188 } 189 190 qp_ctx->private_qp_ctx = (void *)fo_qp_ctx; 191 192 return 0; 193 } 194 195 static int 196 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) 197 { 198 return 0; 199 } 200 201 static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = { 202 worker_attach, 203 worker_detach, 204 scheduler_start, 205 scheduler_stop, 206 scheduler_config_qp, 207 scheduler_create_private_ctx, 208 NULL, /* option_set */ 209 NULL /*option_get */ 210 }; 211 212 static struct rte_cryptodev_scheduler fo_scheduler = { 213 .name = "failover-scheduler", 214 .description = "scheduler which enqueues to the primary worker, " 215 "and only then enqueues to the secondary worker " 216 "upon failing on enqueuing to primary", 217 .mode = CDEV_SCHED_MODE_FAILOVER, 218 .ops = &scheduler_fo_ops 219 }; 220 221 struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler; 222