1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <rte_common.h> 7 #include <rte_dev.h> 8 #include <rte_errno.h> 9 #include <rte_malloc.h> 10 #include <rte_cryptodev_pmd.h> 11 12 #include "bcmfs_device.h" 13 #include "bcmfs_logs.h" 14 #include "bcmfs_qp.h" 15 #include "bcmfs_sym_pmd.h" 16 #include "bcmfs_sym_req.h" 17 18 uint8_t cryptodev_bcmfs_driver_id; 19 20 static int bcmfs_sym_qp_release(struct rte_cryptodev *dev, 21 uint16_t queue_pair_id); 22 23 static int 24 bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev, 25 __rte_unused struct rte_cryptodev_config *config) 26 { 27 return 0; 28 } 29 30 static int 31 bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev) 32 { 33 return 0; 34 } 35 36 static void 37 bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev) 38 { 39 } 40 41 static int 42 bcmfs_sym_dev_close(struct rte_cryptodev *dev) 43 { 44 int i, ret; 45 46 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 47 ret = bcmfs_sym_qp_release(dev, i); 48 if (ret < 0) 49 return ret; 50 } 51 52 return 0; 53 } 54 55 static void 56 bcmfs_sym_dev_info_get(struct rte_cryptodev *dev, 57 struct rte_cryptodev_info *dev_info) 58 { 59 struct bcmfs_sym_dev_private *internals = dev->data->dev_private; 60 struct bcmfs_device *fsdev = internals->fsdev; 61 62 if (dev_info != NULL) { 63 dev_info->driver_id = cryptodev_bcmfs_driver_id; 64 dev_info->feature_flags = dev->feature_flags; 65 dev_info->max_nb_queue_pairs = fsdev->max_hw_qps; 66 /* No limit of number of sessions */ 67 dev_info->sym.max_nb_sessions = 0; 68 } 69 } 70 71 static void 72 bcmfs_sym_stats_get(struct rte_cryptodev *dev, 73 struct rte_cryptodev_stats *stats) 74 { 75 struct bcmfs_qp_stats bcmfs_stats = {0}; 76 struct bcmfs_sym_dev_private *bcmfs_priv; 77 struct bcmfs_device *fsdev; 78 79 if (stats == NULL || dev == NULL) { 80 BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); 81 return; 82 } 83 bcmfs_priv = dev->data->dev_private; 84 fsdev = bcmfs_priv->fsdev; 85 86 bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats); 87 88 stats->enqueued_count = bcmfs_stats.enqueued_count; 89 stats->dequeued_count = bcmfs_stats.dequeued_count; 90 stats->enqueue_err_count = bcmfs_stats.enqueue_err_count; 91 stats->dequeue_err_count = bcmfs_stats.dequeue_err_count; 92 } 93 94 static void 95 bcmfs_sym_stats_reset(struct rte_cryptodev *dev) 96 { 97 struct bcmfs_sym_dev_private *bcmfs_priv; 98 struct bcmfs_device *fsdev; 99 100 if (dev == NULL) { 101 BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev); 102 return; 103 } 104 bcmfs_priv = dev->data->dev_private; 105 fsdev = bcmfs_priv->fsdev; 106 107 bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps); 108 } 109 110 static int 111 bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 112 { 113 struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private; 114 struct bcmfs_qp *qp = (struct bcmfs_qp *) 115 (dev->data->queue_pairs[queue_pair_id]); 116 117 BCMFS_LOG(DEBUG, "Release sym qp %u on device %d", 118 queue_pair_id, dev->data->dev_id); 119 120 rte_mempool_free(qp->sr_mp); 121 122 bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL; 123 124 return bcmfs_qp_release((struct bcmfs_qp **) 125 &dev->data->queue_pairs[queue_pair_id]); 126 } 127 128 static void 129 spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused) 130 { 131 memset(sr, 0, sizeof(*sr)); 132 } 133 134 static void 135 req_pool_obj_init(__rte_unused struct rte_mempool *mp, 136 __rte_unused void *opaque, void *obj, 137 __rte_unused unsigned int obj_idx) 138 { 139 spu_req_init(obj, rte_mempool_virt2iova(obj)); 140 } 141 142 static struct rte_mempool * 143 bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused, 144 uint32_t nobjs, uint16_t qp_id, 145 int socket_id) 146 { 147 char softreq_pool_name[RTE_RING_NAMESIZE]; 148 struct rte_mempool *mp; 149 150 snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d", 151 "bcm_sym", qp_id); 152 153 mp = rte_mempool_create(softreq_pool_name, 154 RTE_ALIGN_MUL_CEIL(nobjs, 64), 155 sizeof(struct bcmfs_sym_request), 156 64, 0, NULL, NULL, req_pool_obj_init, NULL, 157 socket_id, 0); 158 if (mp == NULL) 159 BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d", 160 qp_id, rte_errno); 161 162 return mp; 163 } 164 165 static int 166 bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id, 167 const struct rte_cryptodev_qp_conf *qp_conf, 168 int socket_id) 169 { 170 int ret = 0; 171 struct bcmfs_qp *qp = NULL; 172 struct bcmfs_qp_config bcmfs_qp_conf; 173 174 struct bcmfs_qp **qp_addr = 175 (struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id]; 176 struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private; 177 struct bcmfs_device *fsdev = bcmfs_private->fsdev; 178 179 180 /* If qp is already in use free ring memory and qp metadata. */ 181 if (*qp_addr != NULL) { 182 ret = bcmfs_sym_qp_release(cdev, qp_id); 183 if (ret < 0) 184 return ret; 185 } 186 187 if (qp_id >= fsdev->max_hw_qps) { 188 BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id); 189 return -EINVAL; 190 } 191 192 bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors; 193 bcmfs_qp_conf.socket_id = socket_id; 194 bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ; 195 bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id); 196 bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops; 197 198 ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf); 199 if (ret != 0) 200 return ret; 201 202 qp = (struct bcmfs_qp *)*qp_addr; 203 204 qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors, 205 qp_id, socket_id); 206 if (qp->sr_mp == NULL) 207 return -ENOMEM; 208 209 /* store a link to the qp in the bcmfs_device */ 210 bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr; 211 212 cdev->data->queue_pairs[qp_id] = qp; 213 BCMFS_LOG(NOTICE, "queue %d setup done\n", qp_id); 214 215 return 0; 216 } 217 218 static struct rte_cryptodev_ops crypto_bcmfs_ops = { 219 /* Device related operations */ 220 .dev_configure = bcmfs_sym_dev_config, 221 .dev_start = bcmfs_sym_dev_start, 222 .dev_stop = bcmfs_sym_dev_stop, 223 .dev_close = bcmfs_sym_dev_close, 224 .dev_infos_get = bcmfs_sym_dev_info_get, 225 /* Stats Collection */ 226 .stats_get = bcmfs_sym_stats_get, 227 .stats_reset = bcmfs_sym_stats_reset, 228 /* Queue-Pair management */ 229 .queue_pair_setup = bcmfs_sym_qp_setup, 230 .queue_pair_release = bcmfs_sym_qp_release, 231 }; 232 233 /** Enqueue burst */ 234 static uint16_t 235 bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair, 236 struct rte_crypto_op **ops, 237 uint16_t nb_ops) 238 { 239 int i, j; 240 uint16_t enq = 0; 241 struct bcmfs_sym_request *sreq; 242 struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair; 243 244 if (nb_ops == 0) 245 return 0; 246 247 if (nb_ops > BCMFS_MAX_REQS_BUFF) 248 nb_ops = BCMFS_MAX_REQS_BUFF; 249 250 /* We do not process more than available space */ 251 if (nb_ops > (qp->nb_descriptors - qp->nb_pending_requests)) 252 nb_ops = qp->nb_descriptors - qp->nb_pending_requests; 253 254 for (i = 0; i < nb_ops; i++) { 255 if (rte_mempool_get(qp->sr_mp, (void **)&sreq)) 256 goto enqueue_err; 257 258 /* save rte_crypto_op */ 259 sreq->op = ops[i]; 260 261 /* save context */ 262 qp->infl_msgs[i] = &sreq->msgs; 263 qp->infl_msgs[i]->ctx = (void *)sreq; 264 } 265 /* Send burst request to hw QP */ 266 enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i); 267 268 for (j = enq; j < i; j++) 269 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); 270 271 return enq; 272 273 enqueue_err: 274 for (j = 0; j < i; j++) 275 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); 276 277 return enq; 278 } 279 280 static uint16_t 281 bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair, 282 struct rte_crypto_op **ops, 283 uint16_t nb_ops) 284 { 285 int i; 286 uint16_t deq = 0; 287 unsigned int pkts = 0; 288 struct bcmfs_sym_request *sreq; 289 struct bcmfs_qp *qp = queue_pair; 290 291 if (nb_ops > BCMFS_MAX_REQS_BUFF) 292 nb_ops = BCMFS_MAX_REQS_BUFF; 293 294 deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops); 295 /* get rte_crypto_ops */ 296 for (i = 0; i < deq; i++) { 297 sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx; 298 299 ops[pkts++] = sreq->op; 300 301 rte_mempool_put(qp->sr_mp, sreq); 302 } 303 304 return pkts; 305 } 306 307 /* 308 * An rte_driver is needed in the registration of both the 309 * device and the driver with cryptodev. 310 */ 311 static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD); 312 static const struct rte_driver cryptodev_bcmfs_sym_driver = { 313 .name = bcmfs_sym_drv_name, 314 .alias = bcmfs_sym_drv_name 315 }; 316 317 int 318 bcmfs_sym_dev_create(struct bcmfs_device *fsdev) 319 { 320 struct rte_cryptodev_pmd_init_params init_params = { 321 .name = "", 322 .socket_id = rte_socket_id(), 323 .private_data_size = sizeof(struct bcmfs_sym_dev_private) 324 }; 325 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 326 struct rte_cryptodev *cryptodev; 327 struct bcmfs_sym_dev_private *internals; 328 329 snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", 330 fsdev->name, "sym"); 331 332 /* Populate subset device to use in cryptodev device creation */ 333 fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver; 334 fsdev->sym_rte_dev.numa_node = 0; 335 fsdev->sym_rte_dev.devargs = NULL; 336 337 cryptodev = rte_cryptodev_pmd_create(name, 338 &fsdev->sym_rte_dev, 339 &init_params); 340 if (cryptodev == NULL) 341 return -ENODEV; 342 343 fsdev->sym_rte_dev.name = cryptodev->data->name; 344 cryptodev->driver_id = cryptodev_bcmfs_driver_id; 345 cryptodev->dev_ops = &crypto_bcmfs_ops; 346 347 cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst; 348 cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst; 349 350 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 351 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 352 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 353 354 internals = cryptodev->data->dev_private; 355 internals->fsdev = fsdev; 356 fsdev->sym_dev = internals; 357 358 internals->sym_dev_id = cryptodev->data->dev_id; 359 360 BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d", 361 cryptodev->data->name, internals->sym_dev_id); 362 return 0; 363 } 364 365 int 366 bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev) 367 { 368 struct rte_cryptodev *cryptodev; 369 370 if (fsdev == NULL) 371 return -ENODEV; 372 if (fsdev->sym_dev == NULL) 373 return 0; 374 375 /* free crypto device */ 376 cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id); 377 rte_cryptodev_pmd_destroy(cryptodev); 378 fsdev->sym_rte_dev.name = NULL; 379 fsdev->sym_dev = NULL; 380 381 return 0; 382 } 383 384 static struct cryptodev_driver bcmfs_crypto_drv; 385 RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv, 386 cryptodev_bcmfs_sym_driver, 387 cryptodev_bcmfs_driver_id); 388