xref: /dpdk/drivers/crypto/virtio/virtio_cryptodev.c (revision 72f82c4324fb633d51a6a0f0e237f560d96cb1d7)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
3  */
4 #include <stdbool.h>
5 #include <unistd.h>
6 
7 #include <rte_common.h>
8 #include <rte_errno.h>
9 #include <rte_pci.h>
10 #include <rte_bus_pci.h>
11 #include <rte_cryptodev.h>
12 #include <rte_cryptodev_pmd.h>
13 #include <rte_eal.h>
14 
15 #include "virtio_cryptodev.h"
16 #include "virtqueue.h"
17 #include "virtio_crypto_algs.h"
18 #include "virtio_crypto_capabilities.h"
19 
20 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
21 		struct rte_cryptodev_config *config);
22 static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
23 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
24 static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
25 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
26 		struct rte_cryptodev_info *dev_info);
27 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
28 		struct rte_cryptodev_stats *stats);
29 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev);
30 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
31 		uint16_t queue_pair_id,
32 		const struct rte_cryptodev_qp_conf *qp_conf,
33 		int socket_id);
34 static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
35 		uint16_t queue_pair_id);
36 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
37 static unsigned int virtio_crypto_sym_get_session_private_size(
38 		struct rte_cryptodev *dev);
39 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev,
40 		struct rte_cryptodev_sym_session *sess);
41 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev,
42 		struct rte_crypto_sym_xform *xform,
43 		struct rte_cryptodev_sym_session *session,
44 		struct rte_mempool *mp);
45 
46 /*
47  * The set of PCI devices this driver supports
48  */
49 static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
50 	{ RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
51 				VIRTIO_CRYPTO_PCI_DEVICEID) },
52 	{ .vendor_id = 0, /* sentinel */ },
53 };
54 
55 static const struct rte_cryptodev_capabilities virtio_capabilities[] = {
56 	VIRTIO_SYM_CAPABILITIES,
57 	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
58 };
59 
60 uint8_t cryptodev_virtio_driver_id;
61 
62 #define NUM_ENTRY_SYM_CREATE_SESSION 4
63 
64 static int
65 virtio_crypto_send_command(struct virtqueue *vq,
66 		struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key,
67 		uint8_t *auth_key, struct virtio_crypto_session *session)
68 {
69 	uint8_t idx = 0;
70 	uint8_t needed = 1;
71 	uint32_t head = 0;
72 	uint32_t len_cipher_key = 0;
73 	uint32_t len_auth_key = 0;
74 	uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
75 	uint32_t len_session_input = sizeof(struct virtio_crypto_session_input);
76 	uint32_t len_total = 0;
77 	uint32_t input_offset = 0;
78 	void *virt_addr_started = NULL;
79 	phys_addr_t phys_addr_started;
80 	struct vring_desc *desc;
81 	uint32_t desc_offset;
82 	struct virtio_crypto_session_input *input;
83 	int ret;
84 
85 	PMD_INIT_FUNC_TRACE();
86 
87 	if (session == NULL) {
88 		VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL.");
89 		return -EINVAL;
90 	}
91 	/* cipher only is supported, it is available if auth_key is NULL */
92 	if (!cipher_key) {
93 		VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL.");
94 		return -EINVAL;
95 	}
96 
97 	head = vq->vq_desc_head_idx;
98 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p",
99 					head, vq);
100 
101 	if (vq->vq_free_cnt < needed) {
102 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry");
103 		return -ENOSPC;
104 	}
105 
106 	/* calculate the length of cipher key */
107 	if (cipher_key) {
108 		switch (ctrl->u.sym_create_session.op_type) {
109 		case VIRTIO_CRYPTO_SYM_OP_CIPHER:
110 			len_cipher_key
111 				= ctrl->u.sym_create_session.u.cipher
112 							.para.keylen;
113 			break;
114 		case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
115 			len_cipher_key
116 				= ctrl->u.sym_create_session.u.chain
117 					.para.cipher_param.keylen;
118 			break;
119 		default:
120 			VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type");
121 			return -EINVAL;
122 		}
123 	}
124 
125 	/* calculate the length of auth key */
126 	if (auth_key) {
127 		len_auth_key =
128 			ctrl->u.sym_create_session.u.chain.para.u.mac_param
129 				.auth_key_len;
130 	}
131 
132 	/*
133 	 * malloc memory to store indirect vring_desc entries, including
134 	 * ctrl request, cipher key, auth key, session input and desc vring
135 	 */
136 	desc_offset = len_ctrl_req + len_cipher_key + len_auth_key
137 		+ len_session_input;
138 	virt_addr_started = rte_malloc(NULL,
139 		desc_offset + NUM_ENTRY_SYM_CREATE_SESSION
140 			* sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
141 	if (virt_addr_started == NULL) {
142 		VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory");
143 		return -ENOSPC;
144 	}
145 	phys_addr_started = rte_malloc_virt2iova(virt_addr_started);
146 
147 	/* address to store indirect vring desc entries */
148 	desc = (struct vring_desc *)
149 		((uint8_t *)virt_addr_started + desc_offset);
150 
151 	/*  ctrl req part */
152 	memcpy(virt_addr_started, ctrl, len_ctrl_req);
153 	desc[idx].addr = phys_addr_started;
154 	desc[idx].len = len_ctrl_req;
155 	desc[idx].flags = VRING_DESC_F_NEXT;
156 	desc[idx].next = idx + 1;
157 	idx++;
158 	len_total += len_ctrl_req;
159 	input_offset += len_ctrl_req;
160 
161 	/* cipher key part */
162 	if (len_cipher_key > 0) {
163 		memcpy((uint8_t *)virt_addr_started + len_total,
164 			cipher_key, len_cipher_key);
165 
166 		desc[idx].addr = phys_addr_started + len_total;
167 		desc[idx].len = len_cipher_key;
168 		desc[idx].flags = VRING_DESC_F_NEXT;
169 		desc[idx].next = idx + 1;
170 		idx++;
171 		len_total += len_cipher_key;
172 		input_offset += len_cipher_key;
173 	}
174 
175 	/* auth key part */
176 	if (len_auth_key > 0) {
177 		memcpy((uint8_t *)virt_addr_started + len_total,
178 			auth_key, len_auth_key);
179 
180 		desc[idx].addr = phys_addr_started + len_total;
181 		desc[idx].len = len_auth_key;
182 		desc[idx].flags = VRING_DESC_F_NEXT;
183 		desc[idx].next = idx + 1;
184 		idx++;
185 		len_total += len_auth_key;
186 		input_offset += len_auth_key;
187 	}
188 
189 	/* input part */
190 	input = (struct virtio_crypto_session_input *)
191 		((uint8_t *)virt_addr_started + input_offset);
192 	input->status = VIRTIO_CRYPTO_ERR;
193 	input->session_id = ~0ULL;
194 	desc[idx].addr = phys_addr_started + len_total;
195 	desc[idx].len = len_session_input;
196 	desc[idx].flags = VRING_DESC_F_WRITE;
197 	idx++;
198 
199 	/* use a single desc entry */
200 	vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset;
201 	vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc);
202 	vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
203 	vq->vq_free_cnt--;
204 
205 	vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
206 
207 	vq_update_avail_ring(vq, head);
208 	vq_update_avail_idx(vq);
209 
210 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
211 					vq->vq_queue_index);
212 
213 	virtqueue_notify(vq);
214 
215 	rte_rmb();
216 	while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
217 		rte_rmb();
218 		usleep(100);
219 	}
220 
221 	while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
222 		uint32_t idx, desc_idx, used_idx;
223 		struct vring_used_elem *uep;
224 
225 		used_idx = (uint32_t)(vq->vq_used_cons_idx
226 				& (vq->vq_nentries - 1));
227 		uep = &vq->vq_ring.used->ring[used_idx];
228 		idx = (uint32_t) uep->id;
229 		desc_idx = idx;
230 
231 		while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
232 			desc_idx = vq->vq_ring.desc[desc_idx].next;
233 			vq->vq_free_cnt++;
234 		}
235 
236 		vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
237 		vq->vq_desc_head_idx = idx;
238 
239 		vq->vq_used_cons_idx++;
240 		vq->vq_free_cnt++;
241 	}
242 
243 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
244 			"vq->vq_desc_head_idx=%d",
245 			vq->vq_free_cnt, vq->vq_desc_head_idx);
246 
247 	/* get the result */
248 	if (input->status != VIRTIO_CRYPTO_OK) {
249 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! "
250 				"status=%u, session_id=%" PRIu64 "",
251 				input->status, input->session_id);
252 		rte_free(virt_addr_started);
253 		ret = -1;
254 	} else {
255 		session->session_id = input->session_id;
256 
257 		VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, "
258 				"session_id=%" PRIu64 "", input->session_id);
259 		rte_free(virt_addr_started);
260 		ret = 0;
261 	}
262 
263 	return ret;
264 }
265 
266 void
267 virtio_crypto_queue_release(struct virtqueue *vq)
268 {
269 	struct virtio_crypto_hw *hw;
270 
271 	PMD_INIT_FUNC_TRACE();
272 
273 	if (vq) {
274 		hw = vq->hw;
275 		/* Select and deactivate the queue */
276 		VTPCI_OPS(hw)->del_queue(hw, vq);
277 
278 		rte_memzone_free(vq->mz);
279 		rte_mempool_free(vq->mpool);
280 		rte_free(vq);
281 	}
282 }
283 
284 #define MPOOL_MAX_NAME_SZ 32
285 
286 int
287 virtio_crypto_queue_setup(struct rte_cryptodev *dev,
288 		int queue_type,
289 		uint16_t vtpci_queue_idx,
290 		uint16_t nb_desc,
291 		int socket_id,
292 		struct virtqueue **pvq)
293 {
294 	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
295 	char mpool_name[MPOOL_MAX_NAME_SZ];
296 	const struct rte_memzone *mz;
297 	unsigned int vq_size, size;
298 	struct virtio_crypto_hw *hw = dev->data->dev_private;
299 	struct virtqueue *vq = NULL;
300 	uint32_t i = 0;
301 	uint32_t j;
302 
303 	PMD_INIT_FUNC_TRACE();
304 
305 	VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
306 
307 	/*
308 	 * Read the virtqueue size from the Queue Size field
309 	 * Always power of 2 and if 0 virtqueue does not exist
310 	 */
311 	vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
312 	if (vq_size == 0) {
313 		VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
314 		return -EINVAL;
315 	}
316 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
317 
318 	if (!rte_is_power_of_2(vq_size)) {
319 		VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
320 		return -EINVAL;
321 	}
322 
323 	if (queue_type == VTCRYPTO_DATAQ) {
324 		snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
325 				dev->data->dev_id, vtpci_queue_idx);
326 		snprintf(mpool_name, sizeof(mpool_name),
327 				"dev%d_dataqueue%d_mpool",
328 				dev->data->dev_id, vtpci_queue_idx);
329 	} else if (queue_type == VTCRYPTO_CTRLQ) {
330 		snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
331 				dev->data->dev_id);
332 		snprintf(mpool_name, sizeof(mpool_name),
333 				"dev%d_controlqueue_mpool",
334 				dev->data->dev_id);
335 	}
336 	size = RTE_ALIGN_CEIL(sizeof(*vq) +
337 				vq_size * sizeof(struct vq_desc_extra),
338 				RTE_CACHE_LINE_SIZE);
339 	vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
340 				socket_id);
341 	if (vq == NULL) {
342 		VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
343 		return -ENOMEM;
344 	}
345 
346 	if (queue_type == VTCRYPTO_DATAQ) {
347 		/* pre-allocate a mempool and use it in the data plane to
348 		 * improve performance
349 		 */
350 		vq->mpool = rte_mempool_lookup(mpool_name);
351 		if (vq->mpool == NULL)
352 			vq->mpool = rte_mempool_create(mpool_name,
353 					vq_size,
354 					sizeof(struct virtio_crypto_op_cookie),
355 					RTE_CACHE_LINE_SIZE, 0,
356 					NULL, NULL, NULL, NULL, socket_id,
357 					0);
358 		if (!vq->mpool) {
359 			VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
360 					"Cannot create mempool");
361 			goto mpool_create_err;
362 		}
363 		for (i = 0; i < vq_size; i++) {
364 			vq->vq_descx[i].cookie =
365 				rte_zmalloc("crypto PMD op cookie pointer",
366 					sizeof(struct virtio_crypto_op_cookie),
367 					RTE_CACHE_LINE_SIZE);
368 			if (vq->vq_descx[i].cookie == NULL) {
369 				VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
370 						"alloc mem for cookie");
371 				goto cookie_alloc_err;
372 			}
373 		}
374 	}
375 
376 	vq->hw = hw;
377 	vq->dev_id = dev->data->dev_id;
378 	vq->vq_queue_index = vtpci_queue_idx;
379 	vq->vq_nentries = vq_size;
380 
381 	/*
382 	 * Using part of the vring entries is permitted, but the maximum
383 	 * is vq_size
384 	 */
385 	if (nb_desc == 0 || nb_desc > vq_size)
386 		nb_desc = vq_size;
387 	vq->vq_free_cnt = nb_desc;
388 
389 	/*
390 	 * Reserve a memzone for vring elements
391 	 */
392 	size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
393 	vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
394 	VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
395 			(queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
396 			size, vq->vq_ring_size);
397 
398 	mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
399 			socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
400 	if (mz == NULL) {
401 		if (rte_errno == EEXIST)
402 			mz = rte_memzone_lookup(vq_name);
403 		if (mz == NULL) {
404 			VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
405 			goto mz_reserve_err;
406 		}
407 	}
408 
409 	/*
410 	 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
411 	 * and only accepts 32 bit page frame number.
412 	 * Check if the allocated physical memory exceeds 16TB.
413 	 */
414 	if ((mz->iova + vq->vq_ring_size - 1)
415 				>> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
416 		VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
417 					"above 16TB!");
418 		goto vring_addr_err;
419 	}
420 
421 	memset(mz->addr, 0, sizeof(mz->len));
422 	vq->mz = mz;
423 	vq->vq_ring_mem = mz->iova;
424 	vq->vq_ring_virt_mem = mz->addr;
425 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
426 					(uint64_t)mz->iova);
427 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
428 					(uint64_t)(uintptr_t)mz->addr);
429 
430 	*pvq = vq;
431 
432 	return 0;
433 
434 vring_addr_err:
435 	rte_memzone_free(mz);
436 mz_reserve_err:
437 cookie_alloc_err:
438 	rte_mempool_free(vq->mpool);
439 	if (i != 0) {
440 		for (j = 0; j < i; j++)
441 			rte_free(vq->vq_descx[j].cookie);
442 	}
443 mpool_create_err:
444 	rte_free(vq);
445 	return -ENOMEM;
446 }
447 
448 static int
449 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
450 {
451 	int ret;
452 	struct virtqueue *vq;
453 	struct virtio_crypto_hw *hw = dev->data->dev_private;
454 
455 	/* if virtio device has started, do not touch the virtqueues */
456 	if (dev->data->dev_started)
457 		return 0;
458 
459 	PMD_INIT_FUNC_TRACE();
460 
461 	ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
462 			0, SOCKET_ID_ANY, &vq);
463 	if (ret < 0) {
464 		VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
465 		return ret;
466 	}
467 
468 	hw->cvq = vq;
469 
470 	return 0;
471 }
472 
473 static void
474 virtio_crypto_free_queues(struct rte_cryptodev *dev)
475 {
476 	unsigned int i;
477 	struct virtio_crypto_hw *hw = dev->data->dev_private;
478 
479 	PMD_INIT_FUNC_TRACE();
480 
481 	/* control queue release */
482 	virtio_crypto_queue_release(hw->cvq);
483 
484 	/* data queue release */
485 	for (i = 0; i < hw->max_dataqueues; i++)
486 		virtio_crypto_queue_release(dev->data->queue_pairs[i]);
487 }
488 
489 static int
490 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
491 {
492 	return 0;
493 }
494 
495 /*
496  * dev_ops for virtio, bare necessities for basic operation
497  */
498 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
499 	/* Device related operations */
500 	.dev_configure			 = virtio_crypto_dev_configure,
501 	.dev_start			 = virtio_crypto_dev_start,
502 	.dev_stop			 = virtio_crypto_dev_stop,
503 	.dev_close			 = virtio_crypto_dev_close,
504 	.dev_infos_get			 = virtio_crypto_dev_info_get,
505 
506 	.stats_get			 = virtio_crypto_dev_stats_get,
507 	.stats_reset			 = virtio_crypto_dev_stats_reset,
508 
509 	.queue_pair_setup                = virtio_crypto_qp_setup,
510 	.queue_pair_release              = virtio_crypto_qp_release,
511 
512 	/* Crypto related operations */
513 	.sym_session_get_size		= virtio_crypto_sym_get_session_private_size,
514 	.sym_session_configure		= virtio_crypto_sym_configure_session,
515 	.sym_session_clear		= virtio_crypto_sym_clear_session
516 };
517 
518 static void
519 virtio_crypto_update_stats(struct rte_cryptodev *dev,
520 		struct rte_cryptodev_stats *stats)
521 {
522 	unsigned int i;
523 	struct virtio_crypto_hw *hw = dev->data->dev_private;
524 
525 	PMD_INIT_FUNC_TRACE();
526 
527 	if (stats == NULL) {
528 		VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer");
529 		return;
530 	}
531 
532 	for (i = 0; i < hw->max_dataqueues; i++) {
533 		const struct virtqueue *data_queue
534 			= dev->data->queue_pairs[i];
535 		if (data_queue == NULL)
536 			continue;
537 
538 		stats->enqueued_count += data_queue->packets_sent_total;
539 		stats->enqueue_err_count += data_queue->packets_sent_failed;
540 
541 		stats->dequeued_count += data_queue->packets_received_total;
542 		stats->dequeue_err_count
543 			+= data_queue->packets_received_failed;
544 	}
545 }
546 
547 static void
548 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
549 		struct rte_cryptodev_stats *stats)
550 {
551 	PMD_INIT_FUNC_TRACE();
552 
553 	virtio_crypto_update_stats(dev, stats);
554 }
555 
556 static void
557 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev)
558 {
559 	unsigned int i;
560 	struct virtio_crypto_hw *hw = dev->data->dev_private;
561 
562 	PMD_INIT_FUNC_TRACE();
563 
564 	for (i = 0; i < hw->max_dataqueues; i++) {
565 		struct virtqueue *data_queue = dev->data->queue_pairs[i];
566 		if (data_queue == NULL)
567 			continue;
568 
569 		data_queue->packets_sent_total = 0;
570 		data_queue->packets_sent_failed = 0;
571 
572 		data_queue->packets_received_total = 0;
573 		data_queue->packets_received_failed = 0;
574 	}
575 }
576 
577 static int
578 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
579 		const struct rte_cryptodev_qp_conf *qp_conf,
580 		int socket_id)
581 {
582 	int ret;
583 	struct virtqueue *vq;
584 
585 	PMD_INIT_FUNC_TRACE();
586 
587 	/* if virtio dev is started, do not touch the virtqueues */
588 	if (dev->data->dev_started)
589 		return 0;
590 
591 	ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
592 			qp_conf->nb_descriptors, socket_id, &vq);
593 	if (ret < 0) {
594 		VIRTIO_CRYPTO_INIT_LOG_ERR(
595 			"virtio crypto data queue initialization failed\n");
596 		return ret;
597 	}
598 
599 	dev->data->queue_pairs[queue_pair_id] = vq;
600 
601 	return 0;
602 }
603 
604 static int
605 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
606 {
607 	struct virtqueue *vq
608 		= (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
609 
610 	PMD_INIT_FUNC_TRACE();
611 
612 	if (vq == NULL) {
613 		VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
614 		return 0;
615 	}
616 
617 	virtio_crypto_queue_release(vq);
618 	return 0;
619 }
620 
621 static int
622 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
623 {
624 	uint64_t host_features;
625 
626 	PMD_INIT_FUNC_TRACE();
627 
628 	/* Prepare guest_features: feature that driver wants to support */
629 	VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
630 		req_features);
631 
632 	/* Read device(host) feature bits */
633 	host_features = VTPCI_OPS(hw)->get_features(hw);
634 	VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
635 		host_features);
636 
637 	/*
638 	 * Negotiate features: Subset of device feature bits are written back
639 	 * guest feature bits.
640 	 */
641 	hw->guest_features = req_features;
642 	hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
643 							host_features);
644 	VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
645 		hw->guest_features);
646 
647 	if (hw->modern) {
648 		if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
649 			VIRTIO_CRYPTO_INIT_LOG_ERR(
650 				"VIRTIO_F_VERSION_1 features is not enabled.");
651 			return -1;
652 		}
653 		vtpci_cryptodev_set_status(hw,
654 			VIRTIO_CONFIG_STATUS_FEATURES_OK);
655 		if (!(vtpci_cryptodev_get_status(hw) &
656 			VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
657 			VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
658 						"status!");
659 			return -1;
660 		}
661 	}
662 
663 	hw->req_guest_features = req_features;
664 
665 	return 0;
666 }
667 
668 /* reset device and renegotiate features if needed */
669 static int
670 virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
671 	uint64_t req_features)
672 {
673 	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
674 	struct virtio_crypto_config local_config;
675 	struct virtio_crypto_config *config = &local_config;
676 
677 	PMD_INIT_FUNC_TRACE();
678 
679 	/* Reset the device although not necessary at startup */
680 	vtpci_cryptodev_reset(hw);
681 
682 	/* Tell the host we've noticed this device. */
683 	vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
684 
685 	/* Tell the host we've known how to drive the device. */
686 	vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
687 	if (virtio_negotiate_features(hw, req_features) < 0)
688 		return -1;
689 
690 	/* Get status of the device */
691 	vtpci_read_cryptodev_config(hw,
692 		offsetof(struct virtio_crypto_config, status),
693 		&config->status, sizeof(config->status));
694 	if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
695 		VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
696 				"not ready");
697 		return -1;
698 	}
699 
700 	/* Get number of data queues */
701 	vtpci_read_cryptodev_config(hw,
702 		offsetof(struct virtio_crypto_config, max_dataqueues),
703 		&config->max_dataqueues,
704 		sizeof(config->max_dataqueues));
705 	hw->max_dataqueues = config->max_dataqueues;
706 
707 	VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
708 		hw->max_dataqueues);
709 
710 	return 0;
711 }
712 
713 /*
714  * This function is based on probe() function
715  * It returns 0 on success.
716  */
717 static int
718 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
719 		struct rte_cryptodev_pmd_init_params *init_params)
720 {
721 	struct rte_cryptodev *cryptodev;
722 	struct virtio_crypto_hw *hw;
723 
724 	PMD_INIT_FUNC_TRACE();
725 
726 	cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
727 					init_params);
728 	if (cryptodev == NULL)
729 		return -ENODEV;
730 
731 	cryptodev->driver_id = cryptodev_virtio_driver_id;
732 	cryptodev->dev_ops = &virtio_crypto_dev_ops;
733 
734 	cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
735 	cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;
736 
737 	cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
738 		RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
739 		RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
740 
741 	hw = cryptodev->data->dev_private;
742 	hw->dev_id = cryptodev->data->dev_id;
743 	hw->virtio_dev_capabilities = virtio_capabilities;
744 
745 	VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
746 		cryptodev->data->dev_id, pci_dev->id.vendor_id,
747 		pci_dev->id.device_id);
748 
749 	/* pci device init */
750 	if (vtpci_cryptodev_init(pci_dev, hw))
751 		return -1;
752 
753 	if (virtio_crypto_init_device(cryptodev,
754 			VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
755 		return -1;
756 
757 	return 0;
758 }
759 
760 static int
761 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
762 {
763 	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
764 
765 	PMD_INIT_FUNC_TRACE();
766 
767 	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
768 		return -EPERM;
769 
770 	if (cryptodev->data->dev_started) {
771 		virtio_crypto_dev_stop(cryptodev);
772 		virtio_crypto_dev_close(cryptodev);
773 	}
774 
775 	cryptodev->dev_ops = NULL;
776 	cryptodev->enqueue_burst = NULL;
777 	cryptodev->dequeue_burst = NULL;
778 
779 	/* release control queue */
780 	virtio_crypto_queue_release(hw->cvq);
781 
782 	rte_free(cryptodev->data);
783 	cryptodev->data = NULL;
784 
785 	VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
786 
787 	return 0;
788 }
789 
790 static int
791 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
792 	struct rte_cryptodev_config *config __rte_unused)
793 {
794 	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
795 
796 	PMD_INIT_FUNC_TRACE();
797 
798 	if (virtio_crypto_init_device(cryptodev,
799 			VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
800 		return -1;
801 
802 	/* setup control queue
803 	 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
804 	 * config->max_dataqueues is the control queue
805 	 */
806 	if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
807 		VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
808 		return -1;
809 	}
810 	virtio_crypto_ctrlq_start(cryptodev);
811 
812 	return 0;
813 }
814 
815 static void
816 virtio_crypto_dev_stop(struct rte_cryptodev *dev)
817 {
818 	struct virtio_crypto_hw *hw = dev->data->dev_private;
819 
820 	PMD_INIT_FUNC_TRACE();
821 	VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
822 
823 	vtpci_cryptodev_reset(hw);
824 
825 	virtio_crypto_dev_free_mbufs(dev);
826 	virtio_crypto_free_queues(dev);
827 
828 	dev->data->dev_started = 0;
829 }
830 
831 static int
832 virtio_crypto_dev_start(struct rte_cryptodev *dev)
833 {
834 	struct virtio_crypto_hw *hw = dev->data->dev_private;
835 
836 	if (dev->data->dev_started)
837 		return 0;
838 
839 	/* Do final configuration before queue engine starts */
840 	virtio_crypto_dataq_start(dev);
841 	vtpci_cryptodev_reinit_complete(hw);
842 
843 	dev->data->dev_started = 1;
844 
845 	return 0;
846 }
847 
848 static void
849 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
850 {
851 	uint32_t i;
852 	struct virtio_crypto_hw *hw = dev->data->dev_private;
853 
854 	for (i = 0; i < hw->max_dataqueues; i++) {
855 		VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
856 			"and unused buf", i);
857 		VIRTQUEUE_DUMP((struct virtqueue *)
858 			dev->data->queue_pairs[i]);
859 
860 		VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
861 				i, dev->data->queue_pairs[i]);
862 
863 		virtqueue_detatch_unused(dev->data->queue_pairs[i]);
864 
865 		VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
866 					"unused buf", i);
867 		VIRTQUEUE_DUMP(
868 			(struct virtqueue *)dev->data->queue_pairs[i]);
869 	}
870 }
871 
872 static unsigned int
873 virtio_crypto_sym_get_session_private_size(
874 		struct rte_cryptodev *dev __rte_unused)
875 {
876 	PMD_INIT_FUNC_TRACE();
877 
878 	return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16);
879 }
880 
881 static int
882 virtio_crypto_check_sym_session_paras(
883 		struct rte_cryptodev *dev)
884 {
885 	struct virtio_crypto_hw *hw;
886 
887 	PMD_INIT_FUNC_TRACE();
888 
889 	if (unlikely(dev == NULL)) {
890 		VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL");
891 		return -1;
892 	}
893 	if (unlikely(dev->data == NULL)) {
894 		VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL");
895 		return -1;
896 	}
897 	hw = dev->data->dev_private;
898 	if (unlikely(hw == NULL)) {
899 		VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL");
900 		return -1;
901 	}
902 	if (unlikely(hw->cvq == NULL)) {
903 		VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL");
904 		return -1;
905 	}
906 
907 	return 0;
908 }
909 
910 static int
911 virtio_crypto_check_sym_clear_session_paras(
912 		struct rte_cryptodev *dev,
913 		struct rte_cryptodev_sym_session *sess)
914 {
915 	PMD_INIT_FUNC_TRACE();
916 
917 	if (sess == NULL) {
918 		VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL");
919 		return -1;
920 	}
921 
922 	return virtio_crypto_check_sym_session_paras(dev);
923 }
924 
925 #define NUM_ENTRY_SYM_CLEAR_SESSION 2
926 
927 static void
928 virtio_crypto_sym_clear_session(
929 		struct rte_cryptodev *dev,
930 		struct rte_cryptodev_sym_session *sess)
931 {
932 	struct virtio_crypto_hw *hw;
933 	struct virtqueue *vq;
934 	struct virtio_crypto_session *session;
935 	struct virtio_crypto_op_ctrl_req *ctrl;
936 	struct vring_desc *desc;
937 	uint8_t *status;
938 	uint8_t needed = 1;
939 	uint32_t head;
940 	uint8_t *malloc_virt_addr;
941 	uint64_t malloc_phys_addr;
942 	uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr);
943 	uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
944 	uint32_t desc_offset = len_op_ctrl_req + len_inhdr;
945 
946 	PMD_INIT_FUNC_TRACE();
947 
948 	if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0)
949 		return;
950 
951 	hw = dev->data->dev_private;
952 	vq = hw->cvq;
953 	session = (struct virtio_crypto_session *)get_sym_session_private_data(
954 		sess, cryptodev_virtio_driver_id);
955 	if (session == NULL) {
956 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
957 		return;
958 	}
959 
960 	VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, "
961 			"vq = %p", vq->vq_desc_head_idx, vq);
962 
963 	if (vq->vq_free_cnt < needed) {
964 		VIRTIO_CRYPTO_SESSION_LOG_ERR(
965 				"vq->vq_free_cnt = %d is less than %d, "
966 				"not enough", vq->vq_free_cnt, needed);
967 		return;
968 	}
969 
970 	/*
971 	 * malloc memory to store information of ctrl request op,
972 	 * returned status and desc vring
973 	 */
974 	malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr
975 		+ NUM_ENTRY_SYM_CLEAR_SESSION
976 		* sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
977 	if (malloc_virt_addr == NULL) {
978 		VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room");
979 		return;
980 	}
981 	malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr);
982 
983 	/* assign ctrl request op part */
984 	ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr;
985 	ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION;
986 	/* default data virtqueue is 0 */
987 	ctrl->header.queue_id = 0;
988 	ctrl->u.destroy_session.session_id = session->session_id;
989 
990 	/* status part */
991 	status = &(((struct virtio_crypto_inhdr *)
992 		((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status);
993 	*status = VIRTIO_CRYPTO_ERR;
994 
995 	/* indirect desc vring part */
996 	desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr
997 		+ desc_offset);
998 
999 	/* ctrl request part */
1000 	desc[0].addr = malloc_phys_addr;
1001 	desc[0].len = len_op_ctrl_req;
1002 	desc[0].flags = VRING_DESC_F_NEXT;
1003 	desc[0].next = 1;
1004 
1005 	/* status part */
1006 	desc[1].addr = malloc_phys_addr + len_op_ctrl_req;
1007 	desc[1].len = len_inhdr;
1008 	desc[1].flags = VRING_DESC_F_WRITE;
1009 
1010 	/* use only a single desc entry */
1011 	head = vq->vq_desc_head_idx;
1012 	vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
1013 	vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset;
1014 	vq->vq_ring.desc[head].len
1015 		= NUM_ENTRY_SYM_CLEAR_SESSION
1016 		* sizeof(struct vring_desc);
1017 	vq->vq_free_cnt -= needed;
1018 
1019 	vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
1020 
1021 	vq_update_avail_ring(vq, head);
1022 	vq_update_avail_idx(vq);
1023 
1024 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
1025 					vq->vq_queue_index);
1026 
1027 	virtqueue_notify(vq);
1028 
1029 	rte_rmb();
1030 	while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
1031 		rte_rmb();
1032 		usleep(100);
1033 	}
1034 
1035 	while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
1036 		uint32_t idx, desc_idx, used_idx;
1037 		struct vring_used_elem *uep;
1038 
1039 		used_idx = (uint32_t)(vq->vq_used_cons_idx
1040 				& (vq->vq_nentries - 1));
1041 		uep = &vq->vq_ring.used->ring[used_idx];
1042 		idx = (uint32_t) uep->id;
1043 		desc_idx = idx;
1044 		while (vq->vq_ring.desc[desc_idx].flags
1045 				& VRING_DESC_F_NEXT) {
1046 			desc_idx = vq->vq_ring.desc[desc_idx].next;
1047 			vq->vq_free_cnt++;
1048 		}
1049 
1050 		vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
1051 		vq->vq_desc_head_idx = idx;
1052 		vq->vq_used_cons_idx++;
1053 		vq->vq_free_cnt++;
1054 	}
1055 
1056 	if (*status != VIRTIO_CRYPTO_OK) {
1057 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed "
1058 				"status=%"PRIu32", session_id=%"PRIu64"",
1059 				*status, session->session_id);
1060 		rte_free(malloc_virt_addr);
1061 		return;
1062 	}
1063 
1064 	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
1065 			"vq->vq_desc_head_idx=%d",
1066 			vq->vq_free_cnt, vq->vq_desc_head_idx);
1067 
1068 	VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
1069 			session->session_id);
1070 
1071 	memset(session, 0, sizeof(struct virtio_crypto_session));
1072 	struct rte_mempool *sess_mp = rte_mempool_from_obj(session);
1073 	set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL);
1074 	rte_mempool_put(sess_mp, session);
1075 	rte_free(malloc_virt_addr);
1076 }
1077 
1078 static struct rte_crypto_cipher_xform *
1079 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform)
1080 {
1081 	do {
1082 		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1083 			return &xform->cipher;
1084 
1085 		xform = xform->next;
1086 	} while (xform);
1087 
1088 	return NULL;
1089 }
1090 
1091 static struct rte_crypto_auth_xform *
1092 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform)
1093 {
1094 	do {
1095 		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1096 			return &xform->auth;
1097 
1098 		xform = xform->next;
1099 	} while (xform);
1100 
1101 	return NULL;
1102 }
1103 
1104 /** Get xform chain order */
1105 static int
1106 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform)
1107 {
1108 	if (xform == NULL)
1109 		return -1;
1110 
1111 	/* Cipher Only */
1112 	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1113 			xform->next == NULL)
1114 		return VIRTIO_CRYPTO_CMD_CIPHER;
1115 
1116 	/* Authentication Only */
1117 	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1118 			xform->next == NULL)
1119 		return VIRTIO_CRYPTO_CMD_AUTH;
1120 
1121 	/* Authenticate then Cipher */
1122 	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1123 			xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1124 		return VIRTIO_CRYPTO_CMD_HASH_CIPHER;
1125 
1126 	/* Cipher then Authenticate */
1127 	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1128 			xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1129 		return VIRTIO_CRYPTO_CMD_CIPHER_HASH;
1130 
1131 	return -1;
1132 }
1133 
1134 static int
1135 virtio_crypto_sym_pad_cipher_param(
1136 		struct virtio_crypto_cipher_session_para *para,
1137 		struct rte_crypto_cipher_xform *cipher_xform)
1138 {
1139 	switch (cipher_xform->algo) {
1140 	case RTE_CRYPTO_CIPHER_AES_CBC:
1141 		para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC;
1142 		break;
1143 	default:
1144 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported "
1145 				"Cipher alg %u", cipher_xform->algo);
1146 		return -1;
1147 	}
1148 
1149 	para->keylen = cipher_xform->key.length;
1150 	switch (cipher_xform->op) {
1151 	case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
1152 		para->op = VIRTIO_CRYPTO_OP_ENCRYPT;
1153 		break;
1154 	case RTE_CRYPTO_CIPHER_OP_DECRYPT:
1155 		para->op = VIRTIO_CRYPTO_OP_DECRYPT;
1156 		break;
1157 	default:
1158 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation "
1159 					"parameter");
1160 		return -1;
1161 	}
1162 
1163 	return 0;
1164 }
1165 
1166 static int
1167 virtio_crypto_sym_pad_auth_param(
1168 		struct virtio_crypto_op_ctrl_req *ctrl,
1169 		struct rte_crypto_auth_xform *auth_xform)
1170 {
1171 	uint32_t *algo;
1172 	struct virtio_crypto_alg_chain_session_para *para =
1173 		&(ctrl->u.sym_create_session.u.chain.para);
1174 
1175 	switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) {
1176 	case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN:
1177 		algo = &(para->u.hash_param.algo);
1178 		break;
1179 	case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH:
1180 		algo = &(para->u.mac_param.algo);
1181 		break;
1182 	default:
1183 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u "
1184 			"specified",
1185 			ctrl->u.sym_create_session.u.chain.para.hash_mode);
1186 		return -1;
1187 	}
1188 
1189 	switch (auth_xform->algo) {
1190 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
1191 		*algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1;
1192 		break;
1193 	default:
1194 		VIRTIO_CRYPTO_SESSION_LOG_ERR(
1195 			"Crypto: Undefined Hash algo %u specified",
1196 			auth_xform->algo);
1197 		return -1;
1198 	}
1199 
1200 	return 0;
1201 }
1202 
1203 static int
1204 virtio_crypto_sym_pad_op_ctrl_req(
1205 		struct virtio_crypto_op_ctrl_req *ctrl,
1206 		struct rte_crypto_sym_xform *xform, bool is_chainned,
1207 		uint8_t *cipher_key_data, uint8_t *auth_key_data,
1208 		struct virtio_crypto_session *session)
1209 {
1210 	int ret;
1211 	struct rte_crypto_auth_xform *auth_xform = NULL;
1212 	struct rte_crypto_cipher_xform *cipher_xform = NULL;
1213 
1214 	/* Get cipher xform from crypto xform chain */
1215 	cipher_xform = virtio_crypto_get_cipher_xform(xform);
1216 	if (cipher_xform) {
1217 		if (cipher_xform->key.length > VIRTIO_CRYPTO_MAX_KEY_SIZE) {
1218 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1219 				"cipher key size cannot be longer than %u",
1220 				VIRTIO_CRYPTO_MAX_KEY_SIZE);
1221 			return -1;
1222 		}
1223 		if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) {
1224 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1225 				"cipher IV size cannot be longer than %u",
1226 				VIRTIO_CRYPTO_MAX_IV_SIZE);
1227 			return -1;
1228 		}
1229 		if (is_chainned)
1230 			ret = virtio_crypto_sym_pad_cipher_param(
1231 				&ctrl->u.sym_create_session.u.chain.para
1232 						.cipher_param, cipher_xform);
1233 		else
1234 			ret = virtio_crypto_sym_pad_cipher_param(
1235 				&ctrl->u.sym_create_session.u.cipher.para,
1236 				cipher_xform);
1237 
1238 		if (ret < 0) {
1239 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1240 				"pad cipher parameter failed");
1241 			return -1;
1242 		}
1243 
1244 		memcpy(cipher_key_data, cipher_xform->key.data,
1245 				cipher_xform->key.length);
1246 
1247 		session->iv.offset = cipher_xform->iv.offset;
1248 		session->iv.length = cipher_xform->iv.length;
1249 	}
1250 
1251 	/* Get auth xform from crypto xform chain */
1252 	auth_xform = virtio_crypto_get_auth_xform(xform);
1253 	if (auth_xform) {
1254 		/* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
1255 		struct virtio_crypto_alg_chain_session_para *para =
1256 			&(ctrl->u.sym_create_session.u.chain.para);
1257 		if (auth_xform->key.length) {
1258 			if (auth_xform->key.length >
1259 					VIRTIO_CRYPTO_MAX_KEY_SIZE) {
1260 				VIRTIO_CRYPTO_SESSION_LOG_ERR(
1261 				"auth key size cannot be longer than %u",
1262 					VIRTIO_CRYPTO_MAX_KEY_SIZE);
1263 				return -1;
1264 			}
1265 			para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH;
1266 			para->u.mac_param.auth_key_len =
1267 				(uint32_t)auth_xform->key.length;
1268 			para->u.mac_param.hash_result_len =
1269 				auth_xform->digest_length;
1270 			memcpy(auth_key_data, auth_xform->key.data,
1271 					auth_xform->key.length);
1272 		} else {
1273 			para->hash_mode	= VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN;
1274 			para->u.hash_param.hash_result_len =
1275 				auth_xform->digest_length;
1276 		}
1277 
1278 		ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform);
1279 		if (ret < 0) {
1280 			VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter "
1281 						"failed");
1282 			return -1;
1283 		}
1284 	}
1285 
1286 	return 0;
1287 }
1288 
1289 static int
1290 virtio_crypto_check_sym_configure_session_paras(
1291 		struct rte_cryptodev *dev,
1292 		struct rte_crypto_sym_xform *xform,
1293 		struct rte_cryptodev_sym_session *sym_sess,
1294 		struct rte_mempool *mempool)
1295 {
1296 	if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) ||
1297 		unlikely(mempool == NULL)) {
1298 		VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer");
1299 		return -1;
1300 	}
1301 
1302 	if (virtio_crypto_check_sym_session_paras(dev) < 0)
1303 		return -1;
1304 
1305 	return 0;
1306 }
1307 
1308 static int
1309 virtio_crypto_sym_configure_session(
1310 		struct rte_cryptodev *dev,
1311 		struct rte_crypto_sym_xform *xform,
1312 		struct rte_cryptodev_sym_session *sess,
1313 		struct rte_mempool *mempool)
1314 {
1315 	int ret;
1316 	struct virtio_crypto_session crypto_sess;
1317 	void *session_private = &crypto_sess;
1318 	struct virtio_crypto_session *session;
1319 	struct virtio_crypto_op_ctrl_req *ctrl_req;
1320 	enum virtio_crypto_cmd_id cmd_id;
1321 	uint8_t cipher_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
1322 	uint8_t auth_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
1323 	struct virtio_crypto_hw *hw;
1324 	struct virtqueue *control_vq;
1325 
1326 	PMD_INIT_FUNC_TRACE();
1327 
1328 	ret = virtio_crypto_check_sym_configure_session_paras(dev, xform,
1329 			sess, mempool);
1330 	if (ret < 0) {
1331 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters");
1332 		return ret;
1333 	}
1334 
1335 	if (rte_mempool_get(mempool, &session_private)) {
1336 		VIRTIO_CRYPTO_SESSION_LOG_ERR(
1337 			"Couldn't get object from session mempool");
1338 		return -ENOMEM;
1339 	}
1340 
1341 	session = (struct virtio_crypto_session *)session_private;
1342 	memset(session, 0, sizeof(struct virtio_crypto_session));
1343 	ctrl_req = &session->ctrl;
1344 	ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION;
1345 	/* FIXME: support multiqueue */
1346 	ctrl_req->header.queue_id = 0;
1347 
1348 	hw = dev->data->dev_private;
1349 	control_vq = hw->cvq;
1350 
1351 	cmd_id = virtio_crypto_get_chain_order(xform);
1352 	if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH)
1353 		ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1354 			= VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
1355 	if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER)
1356 		ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1357 			= VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
1358 
1359 	switch (cmd_id) {
1360 	case VIRTIO_CRYPTO_CMD_CIPHER_HASH:
1361 	case VIRTIO_CRYPTO_CMD_HASH_CIPHER:
1362 		ctrl_req->u.sym_create_session.op_type
1363 			= VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING;
1364 
1365 		ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req,
1366 			xform, true, cipher_key_data, auth_key_data, session);
1367 		if (ret < 0) {
1368 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1369 				"padding sym op ctrl req failed");
1370 			goto error_out;
1371 		}
1372 		ret = virtio_crypto_send_command(control_vq, ctrl_req,
1373 			cipher_key_data, auth_key_data, session);
1374 		if (ret < 0) {
1375 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1376 				"create session failed: %d", ret);
1377 			goto error_out;
1378 		}
1379 		break;
1380 	case VIRTIO_CRYPTO_CMD_CIPHER:
1381 		ctrl_req->u.sym_create_session.op_type
1382 			= VIRTIO_CRYPTO_SYM_OP_CIPHER;
1383 		ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform,
1384 			false, cipher_key_data, auth_key_data, session);
1385 		if (ret < 0) {
1386 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1387 				"padding sym op ctrl req failed");
1388 			goto error_out;
1389 		}
1390 		ret = virtio_crypto_send_command(control_vq, ctrl_req,
1391 			cipher_key_data, NULL, session);
1392 		if (ret < 0) {
1393 			VIRTIO_CRYPTO_SESSION_LOG_ERR(
1394 				"create session failed: %d", ret);
1395 			goto error_out;
1396 		}
1397 		break;
1398 	default:
1399 		VIRTIO_CRYPTO_SESSION_LOG_ERR(
1400 			"Unsupported operation chain order parameter");
1401 		goto error_out;
1402 	}
1403 
1404 	set_sym_session_private_data(sess, dev->driver_id,
1405 		session_private);
1406 
1407 	return 0;
1408 
1409 error_out:
1410 	return -1;
1411 }
1412 
1413 static void
1414 virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
1415 		struct rte_cryptodev_info *info)
1416 {
1417 	struct virtio_crypto_hw *hw = dev->data->dev_private;
1418 
1419 	PMD_INIT_FUNC_TRACE();
1420 
1421 	if (info != NULL) {
1422 		info->driver_id = cryptodev_virtio_driver_id;
1423 		info->feature_flags = dev->feature_flags;
1424 		info->max_nb_queue_pairs = hw->max_dataqueues;
1425 		/* No limit of number of sessions */
1426 		info->sym.max_nb_sessions = 0;
1427 		info->capabilities = hw->virtio_dev_capabilities;
1428 	}
1429 }
1430 
1431 static int
1432 crypto_virtio_pci_probe(
1433 	struct rte_pci_driver *pci_drv __rte_unused,
1434 	struct rte_pci_device *pci_dev)
1435 {
1436 	struct rte_cryptodev_pmd_init_params init_params = {
1437 		.name = "",
1438 		.socket_id = pci_dev->device.numa_node,
1439 		.private_data_size = sizeof(struct virtio_crypto_hw)
1440 	};
1441 	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
1442 
1443 	VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
1444 			pci_dev->addr.bus,
1445 			pci_dev->addr.devid,
1446 			pci_dev->addr.function);
1447 
1448 	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
1449 
1450 	return crypto_virtio_create(name, pci_dev, &init_params);
1451 }
1452 
1453 static int
1454 crypto_virtio_pci_remove(
1455 	struct rte_pci_device *pci_dev __rte_unused)
1456 {
1457 	struct rte_cryptodev *cryptodev;
1458 	char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1459 
1460 	if (pci_dev == NULL)
1461 		return -EINVAL;
1462 
1463 	rte_pci_device_name(&pci_dev->addr, cryptodev_name,
1464 			sizeof(cryptodev_name));
1465 
1466 	cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
1467 	if (cryptodev == NULL)
1468 		return -ENODEV;
1469 
1470 	return virtio_crypto_dev_uninit(cryptodev);
1471 }
1472 
1473 static struct rte_pci_driver rte_virtio_crypto_driver = {
1474 	.id_table = pci_id_virtio_crypto_map,
1475 	.drv_flags = 0,
1476 	.probe = crypto_virtio_pci_probe,
1477 	.remove = crypto_virtio_pci_remove
1478 };
1479 
1480 static struct cryptodev_driver virtio_crypto_drv;
1481 
1482 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
1483 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
1484 	rte_virtio_crypto_driver.driver,
1485 	cryptodev_virtio_driver_id);
1486 RTE_LOG_REGISTER(virtio_crypto_logtype_init, pmd.crypto.virtio.init, NOTICE);
1487 RTE_LOG_REGISTER(virtio_crypto_logtype_session, pmd.crypto.virtio.session,
1488 		 NOTICE);
1489 RTE_LOG_REGISTER(virtio_crypto_logtype_rx, pmd.crypto.virtio.rx, NOTICE);
1490 RTE_LOG_REGISTER(virtio_crypto_logtype_tx, pmd.crypto.virtio.tx, NOTICE);
1491 RTE_LOG_REGISTER(virtio_crypto_logtype_driver, pmd.crypto.virtio.driver,
1492 		 NOTICE);
1493