xref: /dpdk/drivers/crypto/octeontx/otx_cryptodev_ops.c (revision 33bcaae5f85ad805ee287bee42013e61a1cff6fa) 
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Cavium, Inc
3  */
4 
5 #include <rte_alarm.h>
6 #include <rte_bus_pci.h>
7 #include <rte_cryptodev.h>
8 #include <rte_cryptodev_pmd.h>
9 #include <rte_errno.h>
10 #include <rte_malloc.h>
11 #include <rte_mempool.h>
12 
13 #include "otx_cryptodev.h"
14 #include "otx_cryptodev_capabilities.h"
15 #include "otx_cryptodev_hw_access.h"
16 #include "otx_cryptodev_mbox.h"
17 #include "otx_cryptodev_ops.h"
18 
19 #include "cpt_pmd_logs.h"
20 #include "cpt_ucode.h"
21 #include "cpt_ucode_asym.h"
22 
23 /* Forward declarations */
24 
25 static int
26 otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id);
27 
28 /* Alarm routines */
29 
30 static void
31 otx_cpt_alarm_cb(void *arg)
32 {
33 	struct cpt_vf *cptvf = arg;
34 	otx_cpt_poll_misc(cptvf);
35 	rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
36 			  otx_cpt_alarm_cb, cptvf);
37 }
38 
39 static int
40 otx_cpt_periodic_alarm_start(void *arg)
41 {
42 	return rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
43 				 otx_cpt_alarm_cb, arg);
44 }
45 
46 static int
47 otx_cpt_periodic_alarm_stop(void *arg)
48 {
49 	return rte_eal_alarm_cancel(otx_cpt_alarm_cb, arg);
50 }
51 
52 /* PMD ops */
53 
54 static int
55 otx_cpt_dev_config(struct rte_cryptodev *dev __rte_unused,
56 		   struct rte_cryptodev_config *config __rte_unused)
57 {
58 	CPT_PMD_INIT_FUNC_TRACE();
59 	return 0;
60 }
61 
62 static int
63 otx_cpt_dev_start(struct rte_cryptodev *c_dev)
64 {
65 	void *cptvf = c_dev->data->dev_private;
66 
67 	CPT_PMD_INIT_FUNC_TRACE();
68 
69 	return otx_cpt_start_device(cptvf);
70 }
71 
72 static void
73 otx_cpt_dev_stop(struct rte_cryptodev *c_dev)
74 {
75 	void *cptvf = c_dev->data->dev_private;
76 
77 	CPT_PMD_INIT_FUNC_TRACE();
78 
79 	otx_cpt_stop_device(cptvf);
80 }
81 
82 static int
83 otx_cpt_dev_close(struct rte_cryptodev *c_dev)
84 {
85 	void *cptvf = c_dev->data->dev_private;
86 	int i, ret;
87 
88 	CPT_PMD_INIT_FUNC_TRACE();
89 
90 	for (i = 0; i < c_dev->data->nb_queue_pairs; i++) {
91 		ret = otx_cpt_que_pair_release(c_dev, i);
92 		if (ret)
93 			return ret;
94 	}
95 
96 	otx_cpt_periodic_alarm_stop(cptvf);
97 	otx_cpt_deinit_device(cptvf);
98 
99 	return 0;
100 }
101 
102 static void
103 otx_cpt_dev_info_get(struct rte_cryptodev *dev, struct rte_cryptodev_info *info)
104 {
105 	CPT_PMD_INIT_FUNC_TRACE();
106 	if (info != NULL) {
107 		info->max_nb_queue_pairs = CPT_NUM_QS_PER_VF;
108 		info->feature_flags = dev->feature_flags;
109 		info->capabilities = otx_get_capabilities(info->feature_flags);
110 		info->sym.max_nb_sessions = 0;
111 		info->driver_id = otx_cryptodev_driver_id;
112 		info->min_mbuf_headroom_req = OTX_CPT_MIN_HEADROOM_REQ;
113 		info->min_mbuf_tailroom_req = OTX_CPT_MIN_TAILROOM_REQ;
114 	}
115 }
116 
117 static void
118 otx_cpt_stats_get(struct rte_cryptodev *dev __rte_unused,
119 		  struct rte_cryptodev_stats *stats __rte_unused)
120 {
121 	CPT_PMD_INIT_FUNC_TRACE();
122 }
123 
124 static void
125 otx_cpt_stats_reset(struct rte_cryptodev *dev __rte_unused)
126 {
127 	CPT_PMD_INIT_FUNC_TRACE();
128 }
129 
130 static int
131 otx_cpt_que_pair_setup(struct rte_cryptodev *dev,
132 		       uint16_t que_pair_id,
133 		       const struct rte_cryptodev_qp_conf *qp_conf,
134 		       int socket_id __rte_unused)
135 {
136 	struct cpt_instance *instance = NULL;
137 	struct rte_pci_device *pci_dev;
138 	int ret = -1;
139 
140 	CPT_PMD_INIT_FUNC_TRACE();
141 
142 	if (dev->data->queue_pairs[que_pair_id] != NULL) {
143 		ret = otx_cpt_que_pair_release(dev, que_pair_id);
144 		if (ret)
145 			return ret;
146 	}
147 
148 	if (qp_conf->nb_descriptors > DEFAULT_CMD_QLEN) {
149 		CPT_LOG_INFO("Number of descriptors too big %d, using default "
150 			     "queue length of %d", qp_conf->nb_descriptors,
151 			     DEFAULT_CMD_QLEN);
152 	}
153 
154 	pci_dev = RTE_DEV_TO_PCI(dev->device);
155 
156 	if (pci_dev->mem_resource[0].addr == NULL) {
157 		CPT_LOG_ERR("PCI mem address null");
158 		return -EIO;
159 	}
160 
161 	ret = otx_cpt_get_resource(dev, 0, &instance, que_pair_id);
162 	if (ret != 0 || instance == NULL) {
163 		CPT_LOG_ERR("Error getting instance handle from device %s : "
164 			    "ret = %d", dev->data->name, ret);
165 		return ret;
166 	}
167 
168 	instance->queue_id = que_pair_id;
169 	instance->sess_mp = qp_conf->mp_session;
170 	instance->sess_mp_priv = qp_conf->mp_session_private;
171 	dev->data->queue_pairs[que_pair_id] = instance;
172 
173 	return 0;
174 }
175 
176 static int
177 otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id)
178 {
179 	struct cpt_instance *instance = dev->data->queue_pairs[que_pair_id];
180 	int ret;
181 
182 	CPT_PMD_INIT_FUNC_TRACE();
183 
184 	ret = otx_cpt_put_resource(instance);
185 	if (ret != 0) {
186 		CPT_LOG_ERR("Error putting instance handle of device %s : "
187 			    "ret = %d", dev->data->name, ret);
188 		return ret;
189 	}
190 
191 	dev->data->queue_pairs[que_pair_id] = NULL;
192 
193 	return 0;
194 }
195 
196 static unsigned int
197 otx_cpt_get_session_size(struct rte_cryptodev *dev __rte_unused)
198 {
199 	return cpt_get_session_size();
200 }
201 
202 static void
203 otx_cpt_session_init(void *sym_sess, uint8_t driver_id)
204 {
205 	struct rte_cryptodev_sym_session *sess = sym_sess;
206 	struct cpt_sess_misc *cpt_sess =
207 	 (struct cpt_sess_misc *) get_sym_session_private_data(sess, driver_id);
208 
209 	CPT_PMD_INIT_FUNC_TRACE();
210 	cpt_sess->ctx_dma_addr = rte_mempool_virt2iova(cpt_sess) +
211 			sizeof(struct cpt_sess_misc);
212 }
213 
214 static int
215 otx_cpt_session_cfg(struct rte_cryptodev *dev,
216 		    struct rte_crypto_sym_xform *xform,
217 		    struct rte_cryptodev_sym_session *sess,
218 		    struct rte_mempool *mempool)
219 {
220 	struct rte_crypto_sym_xform *chain;
221 	void *sess_private_data = NULL;
222 
223 	CPT_PMD_INIT_FUNC_TRACE();
224 
225 	if (cpt_is_algo_supported(xform))
226 		goto err;
227 
228 	if (unlikely(sess == NULL)) {
229 		CPT_LOG_ERR("invalid session struct");
230 		return -EINVAL;
231 	}
232 
233 	if (rte_mempool_get(mempool, &sess_private_data)) {
234 		CPT_LOG_ERR("Could not allocate sess_private_data");
235 		return -ENOMEM;
236 	}
237 
238 	chain = xform;
239 	while (chain) {
240 		switch (chain->type) {
241 		case RTE_CRYPTO_SYM_XFORM_AEAD:
242 			if (fill_sess_aead(chain, sess_private_data))
243 				goto err;
244 			break;
245 		case RTE_CRYPTO_SYM_XFORM_CIPHER:
246 			if (fill_sess_cipher(chain, sess_private_data))
247 				goto err;
248 			break;
249 		case RTE_CRYPTO_SYM_XFORM_AUTH:
250 			if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
251 				if (fill_sess_gmac(chain, sess_private_data))
252 					goto err;
253 			} else {
254 				if (fill_sess_auth(chain, sess_private_data))
255 					goto err;
256 			}
257 			break;
258 		default:
259 			CPT_LOG_ERR("Invalid crypto xform type");
260 			break;
261 		}
262 		chain = chain->next;
263 	}
264 	set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
265 	otx_cpt_session_init(sess, dev->driver_id);
266 	return 0;
267 
268 err:
269 	if (sess_private_data)
270 		rte_mempool_put(mempool, sess_private_data);
271 	return -EPERM;
272 }
273 
274 static void
275 otx_cpt_session_clear(struct rte_cryptodev *dev,
276 		  struct rte_cryptodev_sym_session *sess)
277 {
278 	void *sess_priv = get_sym_session_private_data(sess, dev->driver_id);
279 
280 	CPT_PMD_INIT_FUNC_TRACE();
281 	if (sess_priv) {
282 		memset(sess_priv, 0, otx_cpt_get_session_size(dev));
283 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
284 		set_sym_session_private_data(sess, dev->driver_id, NULL);
285 		rte_mempool_put(sess_mp, sess_priv);
286 	}
287 }
288 
289 static unsigned int
290 otx_cpt_asym_session_size_get(struct rte_cryptodev *dev __rte_unused)
291 {
292 	return sizeof(struct cpt_asym_sess_misc);
293 }
294 
295 static int
296 otx_cpt_asym_session_cfg(struct rte_cryptodev *dev,
297 			 struct rte_crypto_asym_xform *xform __rte_unused,
298 			 struct rte_cryptodev_asym_session *sess,
299 			 struct rte_mempool *pool)
300 {
301 	struct cpt_asym_sess_misc *priv;
302 	int ret;
303 
304 	CPT_PMD_INIT_FUNC_TRACE();
305 
306 	if (rte_mempool_get(pool, (void **)&priv)) {
307 		CPT_LOG_ERR("Could not allocate session private data");
308 		return -ENOMEM;
309 	}
310 
311 	memset(priv, 0, sizeof(struct cpt_asym_sess_misc));
312 
313 	ret = cpt_fill_asym_session_parameters(priv, xform);
314 	if (ret) {
315 		CPT_LOG_ERR("Could not configure session parameters");
316 
317 		/* Return session to mempool */
318 		rte_mempool_put(pool, priv);
319 		return ret;
320 	}
321 
322 	set_asym_session_private_data(sess, dev->driver_id, priv);
323 	return 0;
324 }
325 
326 static void
327 otx_cpt_asym_session_clear(struct rte_cryptodev *dev,
328 			   struct rte_cryptodev_asym_session *sess)
329 {
330 	struct cpt_asym_sess_misc *priv;
331 	struct rte_mempool *sess_mp;
332 
333 	CPT_PMD_INIT_FUNC_TRACE();
334 
335 	priv = get_asym_session_private_data(sess, dev->driver_id);
336 
337 	if (priv == NULL)
338 		return;
339 
340 	/* Free resources allocated during session configure */
341 	cpt_free_asym_session_parameters(priv);
342 	memset(priv, 0, otx_cpt_asym_session_size_get(dev));
343 	sess_mp = rte_mempool_from_obj(priv);
344 	set_asym_session_private_data(sess, dev->driver_id, NULL);
345 	rte_mempool_put(sess_mp, priv);
346 }
347 
348 static __rte_always_inline int32_t __hot
349 otx_cpt_request_enqueue(struct cpt_instance *instance,
350 			struct pending_queue *pqueue,
351 			void *req)
352 {
353 	struct cpt_request_info *user_req = (struct cpt_request_info *)req;
354 
355 	if (unlikely(pqueue->pending_count >= DEFAULT_CMD_QLEN))
356 		return -EAGAIN;
357 
358 	fill_cpt_inst(instance, req);
359 
360 	CPT_LOG_DP_DEBUG("req: %p op: %p ", req, user_req->op);
361 
362 	/* Fill time_out cycles */
363 	user_req->time_out = rte_get_timer_cycles() +
364 			DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
365 	user_req->extra_time = 0;
366 
367 	/* Default mode of software queue */
368 	mark_cpt_inst(instance);
369 
370 	pqueue->rid_queue[pqueue->enq_tail].rid = (uintptr_t)user_req;
371 
372 	/* We will use soft queue length here to limit requests */
373 	MOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN);
374 	pqueue->pending_count += 1;
375 
376 	CPT_LOG_DP_DEBUG("Submitted NB cmd with request: %p "
377 			 "op: %p", user_req, user_req->op);
378 	return 0;
379 }
380 
381 static __rte_always_inline int __hot
382 otx_cpt_enq_single_sym(struct cpt_instance *instance,
383 		       struct rte_crypto_op *op,
384 		       struct pending_queue *pqueue)
385 {
386 	struct cpt_sess_misc *sess;
387 	struct rte_crypto_sym_op *sym_op = op->sym;
388 	void *prep_req, *mdata = NULL;
389 	int ret = 0;
390 	uint64_t cpt_op;
391 
392 	sess = (struct cpt_sess_misc *)
393 			get_sym_session_private_data(sym_op->session,
394 						     otx_cryptodev_driver_id);
395 
396 	cpt_op = sess->cpt_op;
397 
398 	if (likely(cpt_op & CPT_OP_CIPHER_MASK))
399 		ret = fill_fc_params(op, sess, &instance->meta_info, &mdata,
400 				     &prep_req);
401 	else
402 		ret = fill_digest_params(op, sess, &instance->meta_info,
403 					 &mdata, &prep_req);
404 
405 	if (unlikely(ret)) {
406 		CPT_LOG_DP_ERR("prep cryto req : op %p, cpt_op 0x%x "
407 			       "ret 0x%x", op, (unsigned int)cpt_op, ret);
408 		return ret;
409 	}
410 
411 	/* Enqueue prepared instruction to h/w */
412 	ret = otx_cpt_request_enqueue(instance, pqueue, prep_req);
413 
414 	if (unlikely(ret)) {
415 		/* Buffer allocated for request preparation need to be freed */
416 		free_op_meta(mdata, instance->meta_info.pool);
417 		return ret;
418 	}
419 
420 	return 0;
421 }
422 
423 static __rte_always_inline int __hot
424 otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance,
425 				struct rte_crypto_op *op,
426 				struct pending_queue *pqueue)
427 {
428 	struct cpt_sess_misc *sess;
429 	struct rte_crypto_sym_op *sym_op = op->sym;
430 	int ret;
431 	void *sess_t = NULL;
432 	void *sess_private_data_t = NULL;
433 
434 	/* Create tmp session */
435 
436 	if (rte_mempool_get(instance->sess_mp, (void **)&sess_t)) {
437 		ret = -ENOMEM;
438 		goto exit;
439 	}
440 
441 	if (rte_mempool_get(instance->sess_mp_priv,
442 			(void **)&sess_private_data_t)) {
443 		ret = -ENOMEM;
444 		goto free_sess;
445 	}
446 
447 	sess = (struct cpt_sess_misc *)sess_private_data_t;
448 
449 	sess->ctx_dma_addr = rte_mempool_virt2iova(sess) +
450 			sizeof(struct cpt_sess_misc);
451 
452 	ret = instance_session_cfg(sym_op->xform, (void *)sess);
453 	if (unlikely(ret)) {
454 		ret = -EINVAL;
455 		goto free_sess_priv;
456 	}
457 
458 	/* Save tmp session in op */
459 
460 	sym_op->session = (struct rte_cryptodev_sym_session *)sess_t;
461 	set_sym_session_private_data(sym_op->session, otx_cryptodev_driver_id,
462 				     sess_private_data_t);
463 
464 	/* Enqueue op with the tmp session set */
465 	ret = otx_cpt_enq_single_sym(instance, op, pqueue);
466 
467 	if (unlikely(ret))
468 		goto free_sess_priv;
469 
470 	return 0;
471 
472 free_sess_priv:
473 	rte_mempool_put(instance->sess_mp_priv, sess_private_data_t);
474 free_sess:
475 	rte_mempool_put(instance->sess_mp, sess_t);
476 exit:
477 	return ret;
478 }
479 
480 static __rte_always_inline int __hot
481 otx_cpt_enq_single(struct cpt_instance *inst,
482 		   struct rte_crypto_op *op,
483 		   struct pending_queue *pqueue)
484 {
485 	/* Check for the type */
486 
487 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
488 		return otx_cpt_enq_single_sym(inst, op, pqueue);
489 	else if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS))
490 		return otx_cpt_enq_single_sym_sessless(inst, op, pqueue);
491 
492 	/* Should not reach here */
493 	return -EINVAL;
494 }
495 
496 static uint16_t
497 otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
498 {
499 	struct cpt_instance *instance = (struct cpt_instance *)qptr;
500 	uint16_t count;
501 	int ret;
502 	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
503 	struct pending_queue *pqueue = &cptvf->pqueue;
504 
505 	count = DEFAULT_CMD_QLEN - pqueue->pending_count;
506 	if (nb_ops > count)
507 		nb_ops = count;
508 
509 	count = 0;
510 	while (likely(count < nb_ops)) {
511 
512 		/* Enqueue single op */
513 		ret = otx_cpt_enq_single(instance, ops[count], pqueue);
514 
515 		if (unlikely(ret))
516 			break;
517 		count++;
518 	}
519 	otx_cpt_ring_dbell(instance, count);
520 	return count;
521 }
522 
523 static __rte_always_inline void
524 otx_cpt_dequeue_post_process(struct rte_crypto_op *cop, uintptr_t *rsp)
525 {
526 	/* H/w has returned success */
527 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
528 
529 	/* Perform further post processing */
530 
531 	if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
532 		/* Check if auth verify need to be completed */
533 		if (unlikely(rsp[2]))
534 			compl_auth_verify(cop, (uint8_t *)rsp[2], rsp[3]);
535 		return;
536 	}
537 }
538 
539 static uint16_t
540 otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
541 {
542 	struct cpt_instance *instance = (struct cpt_instance *)qptr;
543 	struct cpt_request_info *user_req;
544 	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
545 	struct rid *rid_e;
546 	uint8_t cc[nb_ops];
547 	int i, count, pcount;
548 	uint8_t ret;
549 	int nb_completed;
550 	struct pending_queue *pqueue = &cptvf->pqueue;
551 	struct rte_crypto_op *cop;
552 	void *metabuf;
553 	uintptr_t *rsp;
554 
555 	pcount = pqueue->pending_count;
556 	count = (nb_ops > pcount) ? pcount : nb_ops;
557 
558 	for (i = 0; i < count; i++) {
559 		rid_e = &pqueue->rid_queue[pqueue->deq_head];
560 		user_req = (struct cpt_request_info *)(rid_e->rid);
561 
562 		if (likely((i+1) < count))
563 			rte_prefetch_non_temporal((void *)rid_e[1].rid);
564 
565 		ret = check_nb_command_id(user_req, instance);
566 
567 		if (unlikely(ret == ERR_REQ_PENDING)) {
568 			/* Stop checking for completions */
569 			break;
570 		}
571 
572 		/* Return completion code and op handle */
573 		cc[i] = ret;
574 		ops[i] = user_req->op;
575 
576 		CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d",
577 				 user_req, user_req->op, ret);
578 
579 		MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN);
580 		pqueue->pending_count -= 1;
581 	}
582 
583 	nb_completed = i;
584 
585 	for (i = 0; i < nb_completed; i++) {
586 
587 		rsp = (void *)ops[i];
588 
589 		if (likely((i + 1) < nb_completed))
590 			rte_prefetch0(ops[i+1]);
591 
592 		metabuf = (void *)rsp[0];
593 		cop = (void *)rsp[1];
594 
595 		ops[i] = cop;
596 
597 		/* Check completion code */
598 
599 		if (likely(cc[i] == 0)) {
600 			/* H/w success pkt. Post process */
601 			otx_cpt_dequeue_post_process(cop, rsp);
602 		} else if (cc[i] == ERR_GC_ICV_MISCOMPARE) {
603 			/* auth data mismatch */
604 			cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
605 		} else {
606 			/* Error */
607 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
608 		}
609 
610 		if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
611 			void *sess_private_data_t =
612 				get_sym_session_private_data(cop->sym->session,
613 						otx_cryptodev_driver_id);
614 			memset(sess_private_data_t, 0,
615 					cpt_get_session_size());
616 			memset(cop->sym->session, 0,
617 			rte_cryptodev_sym_get_existing_header_session_size(
618 					cop->sym->session));
619 			rte_mempool_put(instance->sess_mp_priv,
620 					sess_private_data_t);
621 			rte_mempool_put(instance->sess_mp, cop->sym->session);
622 			cop->sym->session = NULL;
623 		}
624 		free_op_meta(metabuf, instance->meta_info.pool);
625 	}
626 
627 	return nb_completed;
628 }
629 
630 static struct rte_cryptodev_ops cptvf_ops = {
631 	/* Device related operations */
632 	.dev_configure = otx_cpt_dev_config,
633 	.dev_start = otx_cpt_dev_start,
634 	.dev_stop = otx_cpt_dev_stop,
635 	.dev_close = otx_cpt_dev_close,
636 	.dev_infos_get = otx_cpt_dev_info_get,
637 
638 	.stats_get = otx_cpt_stats_get,
639 	.stats_reset = otx_cpt_stats_reset,
640 	.queue_pair_setup = otx_cpt_que_pair_setup,
641 	.queue_pair_release = otx_cpt_que_pair_release,
642 	.queue_pair_count = NULL,
643 
644 	/* Crypto related operations */
645 	.sym_session_get_size = otx_cpt_get_session_size,
646 	.sym_session_configure = otx_cpt_session_cfg,
647 	.sym_session_clear = otx_cpt_session_clear,
648 
649 	.asym_session_get_size = otx_cpt_asym_session_size_get,
650 	.asym_session_configure = otx_cpt_asym_session_cfg,
651 	.asym_session_clear = otx_cpt_asym_session_clear,
652 };
653 
654 int
655 otx_cpt_dev_create(struct rte_cryptodev *c_dev)
656 {
657 	struct rte_pci_device *pdev = RTE_DEV_TO_PCI(c_dev->device);
658 	struct cpt_vf *cptvf = NULL;
659 	void *reg_base;
660 	char dev_name[32];
661 	int ret;
662 
663 	if (pdev->mem_resource[0].phys_addr == 0ULL)
664 		return -EIO;
665 
666 	/* for secondary processes, we don't initialise any further as primary
667 	 * has already done this work.
668 	 */
669 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
670 		return 0;
671 
672 	cptvf = rte_zmalloc_socket("otx_cryptodev_private_mem",
673 			sizeof(struct cpt_vf), RTE_CACHE_LINE_SIZE,
674 			rte_socket_id());
675 
676 	if (cptvf == NULL) {
677 		CPT_LOG_ERR("Cannot allocate memory for device private data");
678 		return -ENOMEM;
679 	}
680 
681 	snprintf(dev_name, 32, "%02x:%02x.%x",
682 			pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
683 
684 	reg_base = pdev->mem_resource[0].addr;
685 	if (!reg_base) {
686 		CPT_LOG_ERR("Failed to map BAR0 of %s", dev_name);
687 		ret = -ENODEV;
688 		goto fail;
689 	}
690 
691 	ret = otx_cpt_hw_init(cptvf, pdev, reg_base, dev_name);
692 	if (ret) {
693 		CPT_LOG_ERR("Failed to init cptvf %s", dev_name);
694 		ret = -EIO;
695 		goto fail;
696 	}
697 
698 	switch (cptvf->vftype) {
699 	case OTX_CPT_VF_TYPE_AE:
700 		/* Set asymmetric cpt feature flags */
701 		c_dev->feature_flags = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
702 				RTE_CRYPTODEV_FF_HW_ACCELERATED |
703 				RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT;
704 		break;
705 	case OTX_CPT_VF_TYPE_SE:
706 		/* Set symmetric cpt feature flags */
707 		c_dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
708 				RTE_CRYPTODEV_FF_HW_ACCELERATED |
709 				RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
710 				RTE_CRYPTODEV_FF_IN_PLACE_SGL |
711 				RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
712 				RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT;
713 		break;
714 	default:
715 		/* Feature not supported. Abort */
716 		CPT_LOG_ERR("VF type not supported by %s", dev_name);
717 		ret = -EIO;
718 		goto deinit_dev;
719 	}
720 
721 	/* Start off timer for mailbox interrupts */
722 	otx_cpt_periodic_alarm_start(cptvf);
723 
724 	c_dev->dev_ops = &cptvf_ops;
725 
726 	c_dev->enqueue_burst = otx_cpt_pkt_enqueue;
727 	c_dev->dequeue_burst = otx_cpt_pkt_dequeue;
728 
729 	/* Save dev private data */
730 	c_dev->data->dev_private = cptvf;
731 
732 	return 0;
733 
734 deinit_dev:
735 	otx_cpt_deinit_device(cptvf);
736 
737 fail:
738 	if (cptvf) {
739 		/* Free private data allocated */
740 		rte_free(cptvf);
741 	}
742 
743 	return ret;
744 }
745