xref: /dpdk/drivers/crypto/octeontx/otx_cryptodev_ops.c (revision 25d11a86c56d50947af33d0b79ede622809bd8b9) 
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Cavium, Inc
3  */
4 
5 #include <rte_alarm.h>
6 #include <rte_bus_pci.h>
7 #include <rte_cryptodev.h>
8 #include <rte_cryptodev_pmd.h>
9 #include <rte_malloc.h>
10 
11 #include "cpt_pmd_logs.h"
12 #include "cpt_pmd_ops_helper.h"
13 #include "cpt_ucode.h"
14 #include "cpt_request_mgr.h"
15 
16 #include "otx_cryptodev.h"
17 #include "otx_cryptodev_capabilities.h"
18 #include "otx_cryptodev_hw_access.h"
19 #include "otx_cryptodev_ops.h"
20 
21 static int otx_cryptodev_probe_count;
22 static rte_spinlock_t otx_probe_count_lock = RTE_SPINLOCK_INITIALIZER;
23 
24 static struct rte_mempool *otx_cpt_meta_pool;
25 static int otx_cpt_op_mlen;
26 static int otx_cpt_op_sb_mlen;
27 
28 /* Forward declarations */
29 
30 static int
31 otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id);
32 
33 /*
34  * Initializes global variables used by fast-path code
35  *
36  * @return
37  *   - 0 on success, errcode on error
38  */
39 static int
40 init_global_resources(void)
41 {
42 	/* Get meta len for scatter gather mode */
43 	otx_cpt_op_mlen = cpt_pmd_ops_helper_get_mlen_sg_mode();
44 
45 	/* Extra 4B saved for future considerations */
46 	otx_cpt_op_mlen += 4 * sizeof(uint64_t);
47 
48 	otx_cpt_meta_pool = rte_mempool_create("cpt_metabuf-pool", 4096 * 16,
49 					       otx_cpt_op_mlen, 512, 0,
50 					       NULL, NULL, NULL, NULL,
51 					       SOCKET_ID_ANY, 0);
52 	if (!otx_cpt_meta_pool) {
53 		CPT_LOG_ERR("cpt metabuf pool not created");
54 		return -ENOMEM;
55 	}
56 
57 	/* Get meta len for direct mode */
58 	otx_cpt_op_sb_mlen = cpt_pmd_ops_helper_get_mlen_direct_mode();
59 
60 	/* Extra 4B saved for future considerations */
61 	otx_cpt_op_sb_mlen += 4 * sizeof(uint64_t);
62 
63 	return 0;
64 }
65 
66 void
67 cleanup_global_resources(void)
68 {
69 	/* Take lock */
70 	rte_spinlock_lock(&otx_probe_count_lock);
71 
72 	/* Decrement the cryptodev count */
73 	otx_cryptodev_probe_count--;
74 
75 	/* Free buffers */
76 	if (otx_cpt_meta_pool && otx_cryptodev_probe_count == 0)
77 		rte_mempool_free(otx_cpt_meta_pool);
78 
79 	/* Free lock */
80 	rte_spinlock_unlock(&otx_probe_count_lock);
81 }
82 
83 /* Alarm routines */
84 
85 static void
86 otx_cpt_alarm_cb(void *arg)
87 {
88 	struct cpt_vf *cptvf = arg;
89 	otx_cpt_poll_misc(cptvf);
90 	rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
91 			  otx_cpt_alarm_cb, cptvf);
92 }
93 
94 static int
95 otx_cpt_periodic_alarm_start(void *arg)
96 {
97 	return rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
98 				 otx_cpt_alarm_cb, arg);
99 }
100 
101 static int
102 otx_cpt_periodic_alarm_stop(void *arg)
103 {
104 	return rte_eal_alarm_cancel(otx_cpt_alarm_cb, arg);
105 }
106 
107 /* PMD ops */
108 
109 static int
110 otx_cpt_dev_config(struct rte_cryptodev *dev __rte_unused,
111 		   struct rte_cryptodev_config *config __rte_unused)
112 {
113 	CPT_PMD_INIT_FUNC_TRACE();
114 	return 0;
115 }
116 
117 static int
118 otx_cpt_dev_start(struct rte_cryptodev *c_dev)
119 {
120 	void *cptvf = c_dev->data->dev_private;
121 
122 	CPT_PMD_INIT_FUNC_TRACE();
123 
124 	return otx_cpt_start_device(cptvf);
125 }
126 
127 static void
128 otx_cpt_dev_stop(struct rte_cryptodev *c_dev)
129 {
130 	void *cptvf = c_dev->data->dev_private;
131 
132 	CPT_PMD_INIT_FUNC_TRACE();
133 
134 	otx_cpt_stop_device(cptvf);
135 }
136 
137 static int
138 otx_cpt_dev_close(struct rte_cryptodev *c_dev)
139 {
140 	void *cptvf = c_dev->data->dev_private;
141 	int i, ret;
142 
143 	CPT_PMD_INIT_FUNC_TRACE();
144 
145 	for (i = 0; i < c_dev->data->nb_queue_pairs; i++) {
146 		ret = otx_cpt_que_pair_release(c_dev, i);
147 		if (ret)
148 			return ret;
149 	}
150 
151 	otx_cpt_periodic_alarm_stop(cptvf);
152 	otx_cpt_deinit_device(cptvf);
153 
154 	return 0;
155 }
156 
157 static void
158 otx_cpt_dev_info_get(struct rte_cryptodev *dev, struct rte_cryptodev_info *info)
159 {
160 	CPT_PMD_INIT_FUNC_TRACE();
161 	if (info != NULL) {
162 		info->max_nb_queue_pairs = CPT_NUM_QS_PER_VF;
163 		info->feature_flags = dev->feature_flags;
164 		info->capabilities = otx_get_capabilities();
165 		info->sym.max_nb_sessions = 0;
166 		info->driver_id = otx_cryptodev_driver_id;
167 		info->min_mbuf_headroom_req = OTX_CPT_MIN_HEADROOM_REQ;
168 		info->min_mbuf_tailroom_req = OTX_CPT_MIN_TAILROOM_REQ;
169 	}
170 }
171 
172 static void
173 otx_cpt_stats_get(struct rte_cryptodev *dev __rte_unused,
174 		  struct rte_cryptodev_stats *stats __rte_unused)
175 {
176 	CPT_PMD_INIT_FUNC_TRACE();
177 }
178 
179 static void
180 otx_cpt_stats_reset(struct rte_cryptodev *dev __rte_unused)
181 {
182 	CPT_PMD_INIT_FUNC_TRACE();
183 }
184 
185 static int
186 otx_cpt_que_pair_setup(struct rte_cryptodev *dev,
187 		       uint16_t que_pair_id,
188 		       const struct rte_cryptodev_qp_conf *qp_conf,
189 		       int socket_id __rte_unused)
190 {
191 	void *cptvf = dev->data->dev_private;
192 	struct cpt_instance *instance = NULL;
193 	struct rte_pci_device *pci_dev;
194 	int ret = -1;
195 
196 	CPT_PMD_INIT_FUNC_TRACE();
197 
198 	if (dev->data->queue_pairs[que_pair_id] != NULL) {
199 		ret = otx_cpt_que_pair_release(dev, que_pair_id);
200 		if (ret)
201 			return ret;
202 	}
203 
204 	if (qp_conf->nb_descriptors > DEFAULT_CMD_QLEN) {
205 		CPT_LOG_INFO("Number of descriptors too big %d, using default "
206 			     "queue length of %d", qp_conf->nb_descriptors,
207 			     DEFAULT_CMD_QLEN);
208 	}
209 
210 	pci_dev = RTE_DEV_TO_PCI(dev->device);
211 
212 	if (pci_dev->mem_resource[0].addr == NULL) {
213 		CPT_LOG_ERR("PCI mem address null");
214 		return -EIO;
215 	}
216 
217 	ret = otx_cpt_get_resource(cptvf, 0, &instance);
218 	if (ret != 0 || instance == NULL) {
219 		CPT_LOG_ERR("Error getting instance handle from device %s : "
220 			    "ret = %d", dev->data->name, ret);
221 		return ret;
222 	}
223 
224 	instance->queue_id = que_pair_id;
225 	dev->data->queue_pairs[que_pair_id] = instance;
226 
227 	return 0;
228 }
229 
230 static int
231 otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id)
232 {
233 	struct cpt_instance *instance = dev->data->queue_pairs[que_pair_id];
234 	int ret;
235 
236 	CPT_PMD_INIT_FUNC_TRACE();
237 
238 	ret = otx_cpt_put_resource(instance);
239 	if (ret != 0) {
240 		CPT_LOG_ERR("Error putting instance handle of device %s : "
241 			    "ret = %d", dev->data->name, ret);
242 		return ret;
243 	}
244 
245 	dev->data->queue_pairs[que_pair_id] = NULL;
246 
247 	return 0;
248 }
249 
250 static unsigned int
251 otx_cpt_get_session_size(struct rte_cryptodev *dev __rte_unused)
252 {
253 	return cpt_get_session_size();
254 }
255 
256 static void
257 otx_cpt_session_init(void *sym_sess, uint8_t driver_id)
258 {
259 	struct rte_cryptodev_sym_session *sess = sym_sess;
260 	struct cpt_sess_misc *cpt_sess =
261 	 (struct cpt_sess_misc *) get_sym_session_private_data(sess, driver_id);
262 
263 	CPT_PMD_INIT_FUNC_TRACE();
264 	cpt_sess->ctx_dma_addr = rte_mempool_virt2iova(cpt_sess) +
265 			sizeof(struct cpt_sess_misc);
266 }
267 
268 static int
269 otx_cpt_session_cfg(struct rte_cryptodev *dev,
270 		    struct rte_crypto_sym_xform *xform,
271 		    struct rte_cryptodev_sym_session *sess,
272 		    struct rte_mempool *mempool)
273 {
274 	struct rte_crypto_sym_xform *chain;
275 	void *sess_private_data = NULL;
276 
277 	CPT_PMD_INIT_FUNC_TRACE();
278 
279 	if (cpt_is_algo_supported(xform))
280 		goto err;
281 
282 	if (unlikely(sess == NULL)) {
283 		CPT_LOG_ERR("invalid session struct");
284 		return -EINVAL;
285 	}
286 
287 	if (rte_mempool_get(mempool, &sess_private_data)) {
288 		CPT_LOG_ERR("Could not allocate sess_private_data");
289 		return -ENOMEM;
290 	}
291 
292 	chain = xform;
293 	while (chain) {
294 		switch (chain->type) {
295 		case RTE_CRYPTO_SYM_XFORM_AEAD:
296 			if (fill_sess_aead(chain, sess_private_data))
297 				goto err;
298 			break;
299 		case RTE_CRYPTO_SYM_XFORM_CIPHER:
300 			if (fill_sess_cipher(chain, sess_private_data))
301 				goto err;
302 			break;
303 		case RTE_CRYPTO_SYM_XFORM_AUTH:
304 			if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
305 				if (fill_sess_gmac(chain, sess_private_data))
306 					goto err;
307 			} else {
308 				if (fill_sess_auth(chain, sess_private_data))
309 					goto err;
310 			}
311 			break;
312 		default:
313 			CPT_LOG_ERR("Invalid crypto xform type");
314 			break;
315 		}
316 		chain = chain->next;
317 	}
318 	set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
319 	otx_cpt_session_init(sess, dev->driver_id);
320 	return 0;
321 
322 err:
323 	if (sess_private_data)
324 		rte_mempool_put(mempool, sess_private_data);
325 	return -EPERM;
326 }
327 
328 static void
329 otx_cpt_session_clear(struct rte_cryptodev *dev,
330 		  struct rte_cryptodev_sym_session *sess)
331 {
332 	void *sess_priv = get_sym_session_private_data(sess, dev->driver_id);
333 
334 	CPT_PMD_INIT_FUNC_TRACE();
335 	if (sess_priv) {
336 		memset(sess_priv, 0, otx_cpt_get_session_size(dev));
337 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
338 		set_sym_session_private_data(sess, dev->driver_id, NULL);
339 		rte_mempool_put(sess_mp, sess_priv);
340 	}
341 }
342 
343 static uint16_t
344 otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
345 {
346 	struct cpt_instance *instance = (struct cpt_instance *)qptr;
347 	uint16_t count = 0;
348 	int ret;
349 	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
350 	struct pending_queue *pqueue = &cptvf->pqueue;
351 
352 	count = DEFAULT_CMD_QLEN - pqueue->pending_count;
353 	if (nb_ops > count)
354 		nb_ops = count;
355 
356 	count = 0;
357 	while (likely(count < nb_ops)) {
358 		ret = cpt_pmd_crypto_operation(instance, ops[count], pqueue,
359 						otx_cryptodev_driver_id);
360 		if (unlikely(ret))
361 			break;
362 		count++;
363 	}
364 	otx_cpt_ring_dbell(instance, count);
365 	return count;
366 }
367 
368 static uint16_t
369 otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
370 {
371 	struct cpt_instance *instance = (struct cpt_instance *)qptr;
372 	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
373 	struct pending_queue *pqueue = &cptvf->pqueue;
374 	uint16_t nb_completed, i = 0;
375 	uint8_t compcode[nb_ops];
376 
377 	nb_completed = cpt_dequeue_burst(instance, nb_ops,
378 					 (void **)ops, compcode, pqueue);
379 	while (likely(i < nb_completed)) {
380 		struct rte_crypto_op *cop;
381 		void *metabuf;
382 		uintptr_t *rsp;
383 		uint8_t status;
384 
385 		rsp = (void *)ops[i];
386 		status = compcode[i];
387 		if (likely((i + 1) < nb_completed))
388 			rte_prefetch0(ops[i+1]);
389 		metabuf = (void *)rsp[0];
390 		cop = (void *)rsp[1];
391 
392 		ops[i] = cop;
393 
394 		if (likely(status == 0)) {
395 			if (likely(!rsp[2]))
396 				cop->status =
397 					RTE_CRYPTO_OP_STATUS_SUCCESS;
398 			else
399 				compl_auth_verify(cop, (uint8_t *)rsp[2],
400 						  rsp[3]);
401 		} else if (status == ERR_GC_ICV_MISCOMPARE) {
402 			/*auth data mismatch */
403 			cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
404 		} else {
405 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
406 		}
407 		free_op_meta(metabuf, cptvf->meta_info.cptvf_meta_pool);
408 		i++;
409 	}
410 	return nb_completed;
411 }
412 
413 static struct rte_cryptodev_ops cptvf_ops = {
414 	/* Device related operations */
415 	.dev_configure = otx_cpt_dev_config,
416 	.dev_start = otx_cpt_dev_start,
417 	.dev_stop = otx_cpt_dev_stop,
418 	.dev_close = otx_cpt_dev_close,
419 	.dev_infos_get = otx_cpt_dev_info_get,
420 
421 	.stats_get = otx_cpt_stats_get,
422 	.stats_reset = otx_cpt_stats_reset,
423 	.queue_pair_setup = otx_cpt_que_pair_setup,
424 	.queue_pair_release = otx_cpt_que_pair_release,
425 	.queue_pair_count = NULL,
426 
427 	/* Crypto related operations */
428 	.sym_session_get_size = otx_cpt_get_session_size,
429 	.sym_session_configure = otx_cpt_session_cfg,
430 	.sym_session_clear = otx_cpt_session_clear
431 };
432 
433 static void
434 otx_cpt_common_vars_init(struct cpt_vf *cptvf)
435 {
436 	cptvf->meta_info.cptvf_meta_pool = otx_cpt_meta_pool;
437 	cptvf->meta_info.cptvf_op_mlen = otx_cpt_op_mlen;
438 	cptvf->meta_info.cptvf_op_sb_mlen = otx_cpt_op_sb_mlen;
439 }
440 
441 int
442 otx_cpt_dev_create(struct rte_cryptodev *c_dev)
443 {
444 	struct rte_pci_device *pdev = RTE_DEV_TO_PCI(c_dev->device);
445 	struct cpt_vf *cptvf = NULL;
446 	void *reg_base;
447 	char dev_name[32];
448 	int ret;
449 
450 	if (pdev->mem_resource[0].phys_addr == 0ULL)
451 		return -EIO;
452 
453 	/* for secondary processes, we don't initialise any further as primary
454 	 * has already done this work.
455 	 */
456 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
457 		return 0;
458 
459 	cptvf = rte_zmalloc_socket("otx_cryptodev_private_mem",
460 			sizeof(struct cpt_vf), RTE_CACHE_LINE_SIZE,
461 			rte_socket_id());
462 
463 	if (cptvf == NULL) {
464 		CPT_LOG_ERR("Cannot allocate memory for device private data");
465 		return -ENOMEM;
466 	}
467 
468 	snprintf(dev_name, 32, "%02x:%02x.%x",
469 			pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
470 
471 	reg_base = pdev->mem_resource[0].addr;
472 	if (!reg_base) {
473 		CPT_LOG_ERR("Failed to map BAR0 of %s", dev_name);
474 		ret = -ENODEV;
475 		goto fail;
476 	}
477 
478 	ret = otx_cpt_hw_init(cptvf, pdev, reg_base, dev_name);
479 	if (ret) {
480 		CPT_LOG_ERR("Failed to init cptvf %s", dev_name);
481 		ret = -EIO;
482 		goto fail;
483 	}
484 
485 	/* Start off timer for mailbox interrupts */
486 	otx_cpt_periodic_alarm_start(cptvf);
487 
488 	rte_spinlock_lock(&otx_probe_count_lock);
489 	if (!otx_cryptodev_probe_count) {
490 		ret = init_global_resources();
491 		if (ret) {
492 			rte_spinlock_unlock(&otx_probe_count_lock);
493 			goto init_fail;
494 		}
495 	}
496 	otx_cryptodev_probe_count++;
497 	rte_spinlock_unlock(&otx_probe_count_lock);
498 
499 	/* Initialize data path variables used by common code */
500 	otx_cpt_common_vars_init(cptvf);
501 
502 	c_dev->dev_ops = &cptvf_ops;
503 
504 	c_dev->enqueue_burst = otx_cpt_pkt_enqueue;
505 	c_dev->dequeue_burst = otx_cpt_pkt_dequeue;
506 
507 	c_dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
508 			RTE_CRYPTODEV_FF_HW_ACCELERATED |
509 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
510 			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
511 			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
512 			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT;
513 
514 	/* Save dev private data */
515 	c_dev->data->dev_private = cptvf;
516 
517 	return 0;
518 
519 init_fail:
520 	otx_cpt_periodic_alarm_stop(cptvf);
521 	otx_cpt_deinit_device(cptvf);
522 
523 fail:
524 	if (cptvf) {
525 		/* Free private data allocated */
526 		rte_free(cptvf);
527 	}
528 
529 	return ret;
530 }
531