1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <rte_common.h> 7 #include <dev_driver.h> 8 #include <rte_errno.h> 9 #include <rte_malloc.h> 10 #include <cryptodev_pmd.h> 11 12 #include "bcmfs_device.h" 13 #include "bcmfs_logs.h" 14 #include "bcmfs_qp.h" 15 #include "bcmfs_sym_pmd.h" 16 #include "bcmfs_sym_req.h" 17 #include "bcmfs_sym_session.h" 18 #include "bcmfs_sym_capabilities.h" 19 20 uint8_t cryptodev_bcmfs_driver_id; 21 22 static int bcmfs_sym_qp_release(struct rte_cryptodev *dev, 23 uint16_t queue_pair_id); 24 25 static int 26 bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev, 27 __rte_unused struct rte_cryptodev_config *config) 28 { 29 return 0; 30 } 31 32 static int 33 bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev) 34 { 35 return 0; 36 } 37 38 static void 39 bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev) 40 { 41 } 42 43 static int 44 bcmfs_sym_dev_close(struct rte_cryptodev *dev) 45 { 46 int i, ret; 47 48 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 49 ret = bcmfs_sym_qp_release(dev, i); 50 if (ret < 0) 51 return ret; 52 } 53 54 return 0; 55 } 56 57 static void 58 bcmfs_sym_dev_info_get(struct rte_cryptodev *dev, 59 struct rte_cryptodev_info *dev_info) 60 { 61 struct bcmfs_sym_dev_private *internals = dev->data->dev_private; 62 struct bcmfs_device *fsdev = internals->fsdev; 63 64 if (dev_info != NULL) { 65 dev_info->driver_id = cryptodev_bcmfs_driver_id; 66 dev_info->feature_flags = dev->feature_flags; 67 dev_info->max_nb_queue_pairs = fsdev->max_hw_qps; 68 /* No limit of number of sessions */ 69 dev_info->sym.max_nb_sessions = 0; 70 dev_info->capabilities = bcmfs_sym_get_capabilities(); 71 } 72 } 73 74 static void 75 bcmfs_sym_stats_get(struct rte_cryptodev *dev, 76 struct rte_cryptodev_stats *stats) 77 { 78 struct bcmfs_qp_stats bcmfs_stats = {0}; 79 struct bcmfs_sym_dev_private *bcmfs_priv; 80 struct bcmfs_device *fsdev; 81 82 if (stats == NULL || dev == NULL) { 83 BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); 84 return; 85 } 86 bcmfs_priv = dev->data->dev_private; 87 fsdev = bcmfs_priv->fsdev; 88 89 bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats); 90 91 stats->enqueued_count = bcmfs_stats.enqueued_count; 92 stats->dequeued_count = bcmfs_stats.dequeued_count; 93 stats->enqueue_err_count = bcmfs_stats.enqueue_err_count; 94 stats->dequeue_err_count = bcmfs_stats.dequeue_err_count; 95 } 96 97 static void 98 bcmfs_sym_stats_reset(struct rte_cryptodev *dev) 99 { 100 struct bcmfs_sym_dev_private *bcmfs_priv; 101 struct bcmfs_device *fsdev; 102 103 if (dev == NULL) { 104 BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev); 105 return; 106 } 107 bcmfs_priv = dev->data->dev_private; 108 fsdev = bcmfs_priv->fsdev; 109 110 bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps); 111 } 112 113 static int 114 bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 115 { 116 struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private; 117 struct bcmfs_qp *qp = (struct bcmfs_qp *) 118 (dev->data->queue_pairs[queue_pair_id]); 119 120 BCMFS_LOG(DEBUG, "Release sym qp %u on device %d", 121 queue_pair_id, dev->data->dev_id); 122 123 rte_mempool_free(qp->sr_mp); 124 125 bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL; 126 127 return bcmfs_qp_release((struct bcmfs_qp **) 128 &dev->data->queue_pairs[queue_pair_id]); 129 } 130 131 static void 132 spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused) 133 { 134 memset(sr, 0, sizeof(*sr)); 135 sr->fptr = iova + offsetof(struct bcmfs_sym_request, fmd); 136 sr->optr = iova + offsetof(struct bcmfs_sym_request, omd); 137 sr->dptr = iova + offsetof(struct bcmfs_sym_request, digest); 138 sr->rptr = iova + offsetof(struct bcmfs_sym_request, resp); 139 } 140 141 static void 142 req_pool_obj_init(__rte_unused struct rte_mempool *mp, 143 __rte_unused void *opaque, void *obj, 144 __rte_unused unsigned int obj_idx) 145 { 146 spu_req_init(obj, rte_mempool_virt2iova(obj)); 147 } 148 149 static struct rte_mempool * 150 bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused, 151 uint32_t nobjs, uint16_t qp_id, 152 int socket_id) 153 { 154 char softreq_pool_name[RTE_RING_NAMESIZE]; 155 struct rte_mempool *mp; 156 157 snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d", 158 "bcm_sym", qp_id); 159 160 mp = rte_mempool_create(softreq_pool_name, 161 RTE_ALIGN_MUL_CEIL(nobjs, 64), 162 sizeof(struct bcmfs_sym_request), 163 64, 0, NULL, NULL, req_pool_obj_init, NULL, 164 socket_id, 0); 165 if (mp == NULL) 166 BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d", 167 qp_id, rte_errno); 168 169 return mp; 170 } 171 172 static int 173 bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id, 174 const struct rte_cryptodev_qp_conf *qp_conf, 175 int socket_id) 176 { 177 int ret = 0; 178 struct bcmfs_qp *qp = NULL; 179 struct bcmfs_qp_config bcmfs_qp_conf; 180 181 struct bcmfs_qp **qp_addr = 182 (struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id]; 183 struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private; 184 struct bcmfs_device *fsdev = bcmfs_private->fsdev; 185 186 187 /* If qp is already in use free ring memory and qp metadata. */ 188 if (*qp_addr != NULL) { 189 ret = bcmfs_sym_qp_release(cdev, qp_id); 190 if (ret < 0) 191 return ret; 192 } 193 194 if (qp_id >= fsdev->max_hw_qps) { 195 BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id); 196 return -EINVAL; 197 } 198 199 bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors; 200 bcmfs_qp_conf.socket_id = socket_id; 201 bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ; 202 bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id); 203 bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops; 204 205 ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf); 206 if (ret != 0) 207 return ret; 208 209 qp = (struct bcmfs_qp *)*qp_addr; 210 211 qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors, 212 qp_id, socket_id); 213 if (qp->sr_mp == NULL) 214 return -ENOMEM; 215 216 /* store a link to the qp in the bcmfs_device */ 217 bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr; 218 219 cdev->data->queue_pairs[qp_id] = qp; 220 BCMFS_LOG(NOTICE, "queue %d setup done", qp_id); 221 222 return 0; 223 } 224 225 static struct rte_cryptodev_ops crypto_bcmfs_ops = { 226 /* Device related operations */ 227 .dev_configure = bcmfs_sym_dev_config, 228 .dev_start = bcmfs_sym_dev_start, 229 .dev_stop = bcmfs_sym_dev_stop, 230 .dev_close = bcmfs_sym_dev_close, 231 .dev_infos_get = bcmfs_sym_dev_info_get, 232 /* Stats Collection */ 233 .stats_get = bcmfs_sym_stats_get, 234 .stats_reset = bcmfs_sym_stats_reset, 235 /* Queue-Pair management */ 236 .queue_pair_setup = bcmfs_sym_qp_setup, 237 .queue_pair_release = bcmfs_sym_qp_release, 238 /* Crypto session related operations */ 239 .sym_session_get_size = bcmfs_sym_session_get_private_size, 240 .sym_session_configure = bcmfs_sym_session_configure, 241 .sym_session_clear = bcmfs_sym_session_clear 242 }; 243 244 /** Enqueue burst */ 245 static uint16_t 246 bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair, 247 struct rte_crypto_op **ops, 248 uint16_t nb_ops) 249 { 250 int i, j; 251 int retval; 252 uint16_t enq = 0; 253 struct bcmfs_sym_request *sreq; 254 struct bcmfs_sym_session *sess; 255 struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair; 256 257 if (nb_ops == 0) 258 return 0; 259 260 if (nb_ops > BCMFS_MAX_REQS_BUFF) 261 nb_ops = BCMFS_MAX_REQS_BUFF; 262 263 /* We do not process more than available space */ 264 if (nb_ops > (qp->nb_descriptors - qp->nb_pending_requests)) 265 nb_ops = qp->nb_descriptors - qp->nb_pending_requests; 266 267 for (i = 0; i < nb_ops; i++) { 268 sess = bcmfs_sym_get_session(ops[i]); 269 if (unlikely(sess == NULL)) 270 goto enqueue_err; 271 272 if (rte_mempool_get(qp->sr_mp, (void **)&sreq)) 273 goto enqueue_err; 274 275 /* save rte_crypto_op */ 276 sreq->op = ops[i]; 277 278 /* save context */ 279 qp->infl_msgs[i] = &sreq->msgs; 280 qp->infl_msgs[i]->ctx = (void *)sreq; 281 282 /* pre process the request crypto h/w acceleration */ 283 retval = bcmfs_process_sym_crypto_op(ops[i], sess, sreq); 284 if (unlikely(retval < 0)) 285 goto enqueue_err; 286 } 287 /* Send burst request to hw QP */ 288 enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i); 289 290 for (j = enq; j < i; j++) 291 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); 292 293 return enq; 294 295 enqueue_err: 296 for (j = 0; j < i; j++) 297 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); 298 299 return enq; 300 } 301 302 static void bcmfs_sym_set_request_status(struct rte_crypto_op *op, 303 struct bcmfs_sym_request *out) 304 { 305 if (*out->resp == BCMFS_SYM_RESPONSE_SUCCESS) 306 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 307 else if (*out->resp == BCMFS_SYM_RESPONSE_HASH_TAG_ERROR) 308 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 309 else 310 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 311 } 312 313 static uint16_t 314 bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair, 315 struct rte_crypto_op **ops, 316 uint16_t nb_ops) 317 { 318 int i; 319 uint16_t deq = 0; 320 unsigned int pkts = 0; 321 struct bcmfs_sym_request *sreq; 322 struct bcmfs_qp *qp = queue_pair; 323 324 if (nb_ops > BCMFS_MAX_REQS_BUFF) 325 nb_ops = BCMFS_MAX_REQS_BUFF; 326 327 deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops); 328 /* get rte_crypto_ops */ 329 for (i = 0; i < deq; i++) { 330 sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx; 331 332 /* set the status based on the response from the crypto h/w */ 333 bcmfs_sym_set_request_status(sreq->op, sreq); 334 335 ops[pkts++] = sreq->op; 336 337 rte_mempool_put(qp->sr_mp, sreq); 338 } 339 340 return pkts; 341 } 342 343 /* 344 * An rte_driver is needed in the registration of both the 345 * device and the driver with cryptodev. 346 */ 347 static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD); 348 static const struct rte_driver cryptodev_bcmfs_sym_driver = { 349 .name = bcmfs_sym_drv_name, 350 .alias = bcmfs_sym_drv_name 351 }; 352 353 int 354 bcmfs_sym_dev_create(struct bcmfs_device *fsdev) 355 { 356 struct rte_cryptodev_pmd_init_params init_params = { 357 .name = "", 358 .socket_id = rte_socket_id(), 359 .private_data_size = sizeof(struct bcmfs_sym_dev_private) 360 }; 361 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 362 struct rte_cryptodev *cryptodev; 363 struct bcmfs_sym_dev_private *internals; 364 365 snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", 366 fsdev->name, "sym"); 367 368 /* Populate subset device to use in cryptodev device creation */ 369 fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver; 370 fsdev->sym_rte_dev.numa_node = 0; 371 fsdev->sym_rte_dev.devargs = NULL; 372 373 cryptodev = rte_cryptodev_pmd_create(name, 374 &fsdev->sym_rte_dev, 375 &init_params); 376 if (cryptodev == NULL) 377 return -ENODEV; 378 379 fsdev->sym_rte_dev.name = cryptodev->data->name; 380 cryptodev->driver_id = cryptodev_bcmfs_driver_id; 381 cryptodev->dev_ops = &crypto_bcmfs_ops; 382 383 cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst; 384 cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst; 385 386 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 387 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 388 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 389 390 internals = cryptodev->data->dev_private; 391 internals->fsdev = fsdev; 392 fsdev->sym_dev = internals; 393 394 internals->sym_dev_id = cryptodev->data->dev_id; 395 internals->fsdev_capabilities = bcmfs_sym_get_capabilities(); 396 397 rte_cryptodev_pmd_probing_finish(cryptodev); 398 399 BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d", 400 cryptodev->data->name, internals->sym_dev_id); 401 return 0; 402 } 403 404 int 405 bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev) 406 { 407 struct rte_cryptodev *cryptodev; 408 409 if (fsdev == NULL) 410 return -ENODEV; 411 if (fsdev->sym_dev == NULL) 412 return 0; 413 414 /* free crypto device */ 415 cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id); 416 rte_cryptodev_pmd_destroy(cryptodev); 417 fsdev->sym_rte_dev.name = NULL; 418 fsdev->sym_dev = NULL; 419 420 return 0; 421 } 422 423 static struct cryptodev_driver bcmfs_crypto_drv; 424 RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv, 425 cryptodev_bcmfs_sym_driver, 426 cryptodev_bcmfs_driver_id); 427