1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <rte_common.h> 7 #include <rte_dev.h> 8 #include <rte_errno.h> 9 #include <rte_malloc.h> 10 #include <rte_cryptodev_pmd.h> 11 12 #include "bcmfs_device.h" 13 #include "bcmfs_logs.h" 14 #include "bcmfs_qp.h" 15 #include "bcmfs_sym_pmd.h" 16 #include "bcmfs_sym_req.h" 17 #include "bcmfs_sym_session.h" 18 #include "bcmfs_sym_capabilities.h" 19 20 uint8_t cryptodev_bcmfs_driver_id; 21 22 static int bcmfs_sym_qp_release(struct rte_cryptodev *dev, 23 uint16_t queue_pair_id); 24 25 static int 26 bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev, 27 __rte_unused struct rte_cryptodev_config *config) 28 { 29 return 0; 30 } 31 32 static int 33 bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev) 34 { 35 return 0; 36 } 37 38 static void 39 bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev) 40 { 41 } 42 43 static int 44 bcmfs_sym_dev_close(struct rte_cryptodev *dev) 45 { 46 int i, ret; 47 48 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 49 ret = bcmfs_sym_qp_release(dev, i); 50 if (ret < 0) 51 return ret; 52 } 53 54 return 0; 55 } 56 57 static void 58 bcmfs_sym_dev_info_get(struct rte_cryptodev *dev, 59 struct rte_cryptodev_info *dev_info) 60 { 61 struct bcmfs_sym_dev_private *internals = dev->data->dev_private; 62 struct bcmfs_device *fsdev = internals->fsdev; 63 64 if (dev_info != NULL) { 65 dev_info->driver_id = cryptodev_bcmfs_driver_id; 66 dev_info->feature_flags = dev->feature_flags; 67 dev_info->max_nb_queue_pairs = fsdev->max_hw_qps; 68 /* No limit of number of sessions */ 69 dev_info->sym.max_nb_sessions = 0; 70 dev_info->capabilities = bcmfs_sym_get_capabilities(); 71 } 72 } 73 74 static void 75 bcmfs_sym_stats_get(struct rte_cryptodev *dev, 76 struct rte_cryptodev_stats *stats) 77 { 78 struct bcmfs_qp_stats bcmfs_stats = {0}; 79 struct bcmfs_sym_dev_private *bcmfs_priv; 80 struct bcmfs_device *fsdev; 81 82 if (stats == NULL || dev == NULL) { 83 BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); 84 return; 85 } 86 bcmfs_priv = dev->data->dev_private; 87 fsdev = bcmfs_priv->fsdev; 88 89 bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats); 90 91 stats->enqueued_count = bcmfs_stats.enqueued_count; 92 stats->dequeued_count = bcmfs_stats.dequeued_count; 93 stats->enqueue_err_count = bcmfs_stats.enqueue_err_count; 94 stats->dequeue_err_count = bcmfs_stats.dequeue_err_count; 95 } 96 97 static void 98 bcmfs_sym_stats_reset(struct rte_cryptodev *dev) 99 { 100 struct bcmfs_sym_dev_private *bcmfs_priv; 101 struct bcmfs_device *fsdev; 102 103 if (dev == NULL) { 104 BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev); 105 return; 106 } 107 bcmfs_priv = dev->data->dev_private; 108 fsdev = bcmfs_priv->fsdev; 109 110 bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps); 111 } 112 113 static int 114 bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 115 { 116 struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private; 117 struct bcmfs_qp *qp = (struct bcmfs_qp *) 118 (dev->data->queue_pairs[queue_pair_id]); 119 120 BCMFS_LOG(DEBUG, "Release sym qp %u on device %d", 121 queue_pair_id, dev->data->dev_id); 122 123 rte_mempool_free(qp->sr_mp); 124 125 bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL; 126 127 return bcmfs_qp_release((struct bcmfs_qp **) 128 &dev->data->queue_pairs[queue_pair_id]); 129 } 130 131 static void 132 spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused) 133 { 134 memset(sr, 0, sizeof(*sr)); 135 } 136 137 static void 138 req_pool_obj_init(__rte_unused struct rte_mempool *mp, 139 __rte_unused void *opaque, void *obj, 140 __rte_unused unsigned int obj_idx) 141 { 142 spu_req_init(obj, rte_mempool_virt2iova(obj)); 143 } 144 145 static struct rte_mempool * 146 bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused, 147 uint32_t nobjs, uint16_t qp_id, 148 int socket_id) 149 { 150 char softreq_pool_name[RTE_RING_NAMESIZE]; 151 struct rte_mempool *mp; 152 153 snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d", 154 "bcm_sym", qp_id); 155 156 mp = rte_mempool_create(softreq_pool_name, 157 RTE_ALIGN_MUL_CEIL(nobjs, 64), 158 sizeof(struct bcmfs_sym_request), 159 64, 0, NULL, NULL, req_pool_obj_init, NULL, 160 socket_id, 0); 161 if (mp == NULL) 162 BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d", 163 qp_id, rte_errno); 164 165 return mp; 166 } 167 168 static int 169 bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id, 170 const struct rte_cryptodev_qp_conf *qp_conf, 171 int socket_id) 172 { 173 int ret = 0; 174 struct bcmfs_qp *qp = NULL; 175 struct bcmfs_qp_config bcmfs_qp_conf; 176 177 struct bcmfs_qp **qp_addr = 178 (struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id]; 179 struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private; 180 struct bcmfs_device *fsdev = bcmfs_private->fsdev; 181 182 183 /* If qp is already in use free ring memory and qp metadata. */ 184 if (*qp_addr != NULL) { 185 ret = bcmfs_sym_qp_release(cdev, qp_id); 186 if (ret < 0) 187 return ret; 188 } 189 190 if (qp_id >= fsdev->max_hw_qps) { 191 BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id); 192 return -EINVAL; 193 } 194 195 bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors; 196 bcmfs_qp_conf.socket_id = socket_id; 197 bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ; 198 bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id); 199 bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops; 200 201 ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf); 202 if (ret != 0) 203 return ret; 204 205 qp = (struct bcmfs_qp *)*qp_addr; 206 207 qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors, 208 qp_id, socket_id); 209 if (qp->sr_mp == NULL) 210 return -ENOMEM; 211 212 /* store a link to the qp in the bcmfs_device */ 213 bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr; 214 215 cdev->data->queue_pairs[qp_id] = qp; 216 BCMFS_LOG(NOTICE, "queue %d setup done\n", qp_id); 217 218 return 0; 219 } 220 221 static struct rte_cryptodev_ops crypto_bcmfs_ops = { 222 /* Device related operations */ 223 .dev_configure = bcmfs_sym_dev_config, 224 .dev_start = bcmfs_sym_dev_start, 225 .dev_stop = bcmfs_sym_dev_stop, 226 .dev_close = bcmfs_sym_dev_close, 227 .dev_infos_get = bcmfs_sym_dev_info_get, 228 /* Stats Collection */ 229 .stats_get = bcmfs_sym_stats_get, 230 .stats_reset = bcmfs_sym_stats_reset, 231 /* Queue-Pair management */ 232 .queue_pair_setup = bcmfs_sym_qp_setup, 233 .queue_pair_release = bcmfs_sym_qp_release, 234 /* Crypto session related operations */ 235 .sym_session_get_size = bcmfs_sym_session_get_private_size, 236 .sym_session_configure = bcmfs_sym_session_configure, 237 .sym_session_clear = bcmfs_sym_session_clear 238 }; 239 240 /** Enqueue burst */ 241 static uint16_t 242 bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair, 243 struct rte_crypto_op **ops, 244 uint16_t nb_ops) 245 { 246 int i, j; 247 uint16_t enq = 0; 248 struct bcmfs_sym_request *sreq; 249 struct bcmfs_sym_session *sess; 250 struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair; 251 252 if (nb_ops == 0) 253 return 0; 254 255 if (nb_ops > BCMFS_MAX_REQS_BUFF) 256 nb_ops = BCMFS_MAX_REQS_BUFF; 257 258 /* We do not process more than available space */ 259 if (nb_ops > (qp->nb_descriptors - qp->nb_pending_requests)) 260 nb_ops = qp->nb_descriptors - qp->nb_pending_requests; 261 262 for (i = 0; i < nb_ops; i++) { 263 sess = bcmfs_sym_get_session(ops[i]); 264 if (unlikely(sess == NULL)) 265 goto enqueue_err; 266 267 if (rte_mempool_get(qp->sr_mp, (void **)&sreq)) 268 goto enqueue_err; 269 270 /* save rte_crypto_op */ 271 sreq->op = ops[i]; 272 273 /* save context */ 274 qp->infl_msgs[i] = &sreq->msgs; 275 qp->infl_msgs[i]->ctx = (void *)sreq; 276 } 277 /* Send burst request to hw QP */ 278 enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i); 279 280 for (j = enq; j < i; j++) 281 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); 282 283 return enq; 284 285 enqueue_err: 286 for (j = 0; j < i; j++) 287 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); 288 289 return enq; 290 } 291 292 static uint16_t 293 bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair, 294 struct rte_crypto_op **ops, 295 uint16_t nb_ops) 296 { 297 int i; 298 uint16_t deq = 0; 299 unsigned int pkts = 0; 300 struct bcmfs_sym_request *sreq; 301 struct bcmfs_qp *qp = queue_pair; 302 303 if (nb_ops > BCMFS_MAX_REQS_BUFF) 304 nb_ops = BCMFS_MAX_REQS_BUFF; 305 306 deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops); 307 /* get rte_crypto_ops */ 308 for (i = 0; i < deq; i++) { 309 sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx; 310 311 ops[pkts++] = sreq->op; 312 313 rte_mempool_put(qp->sr_mp, sreq); 314 } 315 316 return pkts; 317 } 318 319 /* 320 * An rte_driver is needed in the registration of both the 321 * device and the driver with cryptodev. 322 */ 323 static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD); 324 static const struct rte_driver cryptodev_bcmfs_sym_driver = { 325 .name = bcmfs_sym_drv_name, 326 .alias = bcmfs_sym_drv_name 327 }; 328 329 int 330 bcmfs_sym_dev_create(struct bcmfs_device *fsdev) 331 { 332 struct rte_cryptodev_pmd_init_params init_params = { 333 .name = "", 334 .socket_id = rte_socket_id(), 335 .private_data_size = sizeof(struct bcmfs_sym_dev_private) 336 }; 337 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 338 struct rte_cryptodev *cryptodev; 339 struct bcmfs_sym_dev_private *internals; 340 341 snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", 342 fsdev->name, "sym"); 343 344 /* Populate subset device to use in cryptodev device creation */ 345 fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver; 346 fsdev->sym_rte_dev.numa_node = 0; 347 fsdev->sym_rte_dev.devargs = NULL; 348 349 cryptodev = rte_cryptodev_pmd_create(name, 350 &fsdev->sym_rte_dev, 351 &init_params); 352 if (cryptodev == NULL) 353 return -ENODEV; 354 355 fsdev->sym_rte_dev.name = cryptodev->data->name; 356 cryptodev->driver_id = cryptodev_bcmfs_driver_id; 357 cryptodev->dev_ops = &crypto_bcmfs_ops; 358 359 cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst; 360 cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst; 361 362 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 363 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 364 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 365 366 internals = cryptodev->data->dev_private; 367 internals->fsdev = fsdev; 368 fsdev->sym_dev = internals; 369 370 internals->sym_dev_id = cryptodev->data->dev_id; 371 internals->fsdev_capabilities = bcmfs_sym_get_capabilities(); 372 373 BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d", 374 cryptodev->data->name, internals->sym_dev_id); 375 return 0; 376 } 377 378 int 379 bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev) 380 { 381 struct rte_cryptodev *cryptodev; 382 383 if (fsdev == NULL) 384 return -ENODEV; 385 if (fsdev->sym_dev == NULL) 386 return 0; 387 388 /* free crypto device */ 389 cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id); 390 rte_cryptodev_pmd_destroy(cryptodev); 391 fsdev->sym_rte_dev.name = NULL; 392 fsdev->sym_dev = NULL; 393 394 return 0; 395 } 396 397 static struct cryptodev_driver bcmfs_crypto_drv; 398 RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv, 399 cryptodev_bcmfs_sym_driver, 400 cryptodev_bcmfs_driver_id); 401