xref: /dpdk/drivers/crypto/openssl/rte_openssl_pmd.c (revision b79e4c00af0e7cfb8601ab0208659d226b82bd10)
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Intel Corporation nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <rte_common.h>
34 #include <rte_hexdump.h>
35 #include <rte_cryptodev.h>
36 #include <rte_cryptodev_pmd.h>
37 #include <rte_cryptodev_vdev.h>
38 #include <rte_vdev.h>
39 #include <rte_malloc.h>
40 #include <rte_cpuflags.h>
41 
42 #include <openssl/evp.h>
43 
44 #include "rte_openssl_pmd_private.h"
45 
46 #define DES_BLOCK_SIZE 8
47 
48 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
49 
50 /*----------------------------------------------------------------------------*/
51 
52 /**
53  * Increment counter by 1
54  * Counter is 64 bit array, big-endian
55  */
56 static void
57 ctr_inc(uint8_t *ctr)
58 {
59 	uint64_t *ctr64 = (uint64_t *)ctr;
60 
61 	*ctr64 = __builtin_bswap64(*ctr64);
62 	(*ctr64)++;
63 	*ctr64 = __builtin_bswap64(*ctr64);
64 }
65 
66 /*
67  *------------------------------------------------------------------------------
68  * Session Prepare
69  *------------------------------------------------------------------------------
70  */
71 
72 /** Get xform chain order */
73 static enum openssl_chain_order
74 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
75 {
76 	enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
77 
78 	if (xform != NULL) {
79 		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
80 			if (xform->next == NULL)
81 				res =  OPENSSL_CHAIN_ONLY_AUTH;
82 			else if (xform->next->type ==
83 					RTE_CRYPTO_SYM_XFORM_CIPHER)
84 				res =  OPENSSL_CHAIN_AUTH_CIPHER;
85 		}
86 		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
87 			if (xform->next == NULL)
88 				res =  OPENSSL_CHAIN_ONLY_CIPHER;
89 			else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
90 				res =  OPENSSL_CHAIN_CIPHER_AUTH;
91 		}
92 		if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
93 			res = OPENSSL_CHAIN_COMBINED;
94 	}
95 
96 	return res;
97 }
98 
99 /** Get session cipher key from input cipher key */
100 static void
101 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
102 {
103 	memcpy(session_key, input_key, keylen);
104 }
105 
106 /** Get key ede 24 bytes standard from input key */
107 static int
108 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
109 {
110 	int res = 0;
111 
112 	/* Initialize keys - 24 bytes: [key1-key2-key3] */
113 	switch (keylen) {
114 	case 24:
115 		memcpy(key_ede, key, 24);
116 		break;
117 	case 16:
118 		/* K3 = K1 */
119 		memcpy(key_ede, key, 16);
120 		memcpy(key_ede + 16, key, 8);
121 		break;
122 	case 8:
123 		/* K1 = K2 = K3 (DES compatibility) */
124 		memcpy(key_ede, key, 8);
125 		memcpy(key_ede + 8, key, 8);
126 		memcpy(key_ede + 16, key, 8);
127 		break;
128 	default:
129 		OPENSSL_LOG_ERR("Unsupported key size");
130 		res = -EINVAL;
131 	}
132 
133 	return res;
134 }
135 
136 /** Get adequate openssl function for input cipher algorithm */
137 static uint8_t
138 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
139 		const EVP_CIPHER **algo)
140 {
141 	int res = 0;
142 
143 	if (algo != NULL) {
144 		switch (sess_algo) {
145 		case RTE_CRYPTO_CIPHER_3DES_CBC:
146 			switch (keylen) {
147 			case 16:
148 				*algo = EVP_des_ede_cbc();
149 				break;
150 			case 24:
151 				*algo = EVP_des_ede3_cbc();
152 				break;
153 			default:
154 				res = -EINVAL;
155 			}
156 			break;
157 		case RTE_CRYPTO_CIPHER_3DES_CTR:
158 			break;
159 		case RTE_CRYPTO_CIPHER_AES_CBC:
160 			switch (keylen) {
161 			case 16:
162 				*algo = EVP_aes_128_cbc();
163 				break;
164 			case 24:
165 				*algo = EVP_aes_192_cbc();
166 				break;
167 			case 32:
168 				*algo = EVP_aes_256_cbc();
169 				break;
170 			default:
171 				res = -EINVAL;
172 			}
173 			break;
174 		case RTE_CRYPTO_CIPHER_AES_CTR:
175 			switch (keylen) {
176 			case 16:
177 				*algo = EVP_aes_128_ctr();
178 				break;
179 			case 24:
180 				*algo = EVP_aes_192_ctr();
181 				break;
182 			case 32:
183 				*algo = EVP_aes_256_ctr();
184 				break;
185 			default:
186 				res = -EINVAL;
187 			}
188 			break;
189 		default:
190 			res = -EINVAL;
191 			break;
192 		}
193 	} else {
194 		res = -EINVAL;
195 	}
196 
197 	return res;
198 }
199 
200 /** Get adequate openssl function for input auth algorithm */
201 static uint8_t
202 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
203 		const EVP_MD **algo)
204 {
205 	int res = 0;
206 
207 	if (algo != NULL) {
208 		switch (sessalgo) {
209 		case RTE_CRYPTO_AUTH_MD5:
210 		case RTE_CRYPTO_AUTH_MD5_HMAC:
211 			*algo = EVP_md5();
212 			break;
213 		case RTE_CRYPTO_AUTH_SHA1:
214 		case RTE_CRYPTO_AUTH_SHA1_HMAC:
215 			*algo = EVP_sha1();
216 			break;
217 		case RTE_CRYPTO_AUTH_SHA224:
218 		case RTE_CRYPTO_AUTH_SHA224_HMAC:
219 			*algo = EVP_sha224();
220 			break;
221 		case RTE_CRYPTO_AUTH_SHA256:
222 		case RTE_CRYPTO_AUTH_SHA256_HMAC:
223 			*algo = EVP_sha256();
224 			break;
225 		case RTE_CRYPTO_AUTH_SHA384:
226 		case RTE_CRYPTO_AUTH_SHA384_HMAC:
227 			*algo = EVP_sha384();
228 			break;
229 		case RTE_CRYPTO_AUTH_SHA512:
230 		case RTE_CRYPTO_AUTH_SHA512_HMAC:
231 			*algo = EVP_sha512();
232 			break;
233 		default:
234 			res = -EINVAL;
235 			break;
236 		}
237 	} else {
238 		res = -EINVAL;
239 	}
240 
241 	return res;
242 }
243 
244 /** Get adequate openssl function for input cipher algorithm */
245 static uint8_t
246 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
247 		const EVP_CIPHER **algo)
248 {
249 	int res = 0;
250 
251 	if (algo != NULL) {
252 		switch (sess_algo) {
253 		case RTE_CRYPTO_AEAD_AES_GCM:
254 			switch (keylen) {
255 			case 16:
256 				*algo = EVP_aes_128_gcm();
257 				break;
258 			case 24:
259 				*algo = EVP_aes_192_gcm();
260 				break;
261 			case 32:
262 				*algo = EVP_aes_256_gcm();
263 				break;
264 			default:
265 				res = -EINVAL;
266 			}
267 			break;
268 		default:
269 			res = -EINVAL;
270 			break;
271 		}
272 	} else {
273 		res = -EINVAL;
274 	}
275 
276 	return res;
277 }
278 
279 /** Set session cipher parameters */
280 static int
281 openssl_set_session_cipher_parameters(struct openssl_session *sess,
282 		const struct rte_crypto_sym_xform *xform)
283 {
284 	/* Select cipher direction */
285 	sess->cipher.direction = xform->cipher.op;
286 	/* Select cipher key */
287 	sess->cipher.key.length = xform->cipher.key.length;
288 
289 	/* Set IV parameters */
290 	sess->iv.offset = xform->cipher.iv.offset;
291 	sess->iv.length = xform->cipher.iv.length;
292 
293 	/* Select cipher algo */
294 	switch (xform->cipher.algo) {
295 	case RTE_CRYPTO_CIPHER_3DES_CBC:
296 	case RTE_CRYPTO_CIPHER_AES_CBC:
297 	case RTE_CRYPTO_CIPHER_AES_CTR:
298 		sess->cipher.mode = OPENSSL_CIPHER_LIB;
299 		sess->cipher.algo = xform->cipher.algo;
300 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
301 
302 		if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
303 				&sess->cipher.evp_algo) != 0)
304 			return -EINVAL;
305 
306 		get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
307 			sess->cipher.key.data);
308 
309 		break;
310 
311 	case RTE_CRYPTO_CIPHER_3DES_CTR:
312 		sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
313 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
314 
315 		if (get_cipher_key_ede(xform->cipher.key.data,
316 				sess->cipher.key.length,
317 				sess->cipher.key.data) != 0)
318 			return -EINVAL;
319 		break;
320 	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
321 		sess->cipher.algo = xform->cipher.algo;
322 		sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
323 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
324 		sess->cipher.evp_algo = EVP_des_cbc();
325 
326 		sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
327 		/* IV will be ECB encrypted whether direction is encrypt or decrypt */
328 		if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
329 				NULL, xform->cipher.key.data, 0) != 1)
330 			return -EINVAL;
331 
332 		get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
333 			sess->cipher.key.data);
334 		break;
335 	default:
336 		sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
337 		return -EINVAL;
338 	}
339 
340 	return 0;
341 }
342 
343 /* Set session auth parameters */
344 static int
345 openssl_set_session_auth_parameters(struct openssl_session *sess,
346 		const struct rte_crypto_sym_xform *xform)
347 {
348 	/* Select auth generate/verify */
349 	sess->auth.operation = xform->auth.op;
350 	sess->auth.algo = xform->auth.algo;
351 
352 	/* Select auth algo */
353 	switch (xform->auth.algo) {
354 	case RTE_CRYPTO_AUTH_AES_GMAC:
355 		sess->chain_order = OPENSSL_CHAIN_COMBINED;
356 
357 		/* Set IV parameters */
358 		sess->iv.offset = xform->auth.iv.offset;
359 		sess->iv.length = xform->auth.iv.length;
360 
361 		/*
362 		 * OpenSSL requires GMAC to be a GCM operation
363 		 * with no cipher data length
364 		 */
365 		sess->cipher.mode = OPENSSL_CIPHER_LIB;
366 		if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
367 			sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
368 		else
369 			sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
370 
371 		sess->cipher.key.length = xform->auth.key.length;
372 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
373 
374 		if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM,
375 				sess->cipher.key.length,
376 				&sess->cipher.evp_algo) != 0)
377 			return -EINVAL;
378 
379 		get_cipher_key(xform->auth.key.data, xform->auth.key.length,
380 			sess->cipher.key.data);
381 
382 		break;
383 
384 	case RTE_CRYPTO_AUTH_MD5:
385 	case RTE_CRYPTO_AUTH_SHA1:
386 	case RTE_CRYPTO_AUTH_SHA224:
387 	case RTE_CRYPTO_AUTH_SHA256:
388 	case RTE_CRYPTO_AUTH_SHA384:
389 	case RTE_CRYPTO_AUTH_SHA512:
390 		sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
391 		if (get_auth_algo(xform->auth.algo,
392 				&sess->auth.auth.evp_algo) != 0)
393 			return -EINVAL;
394 		sess->auth.auth.ctx = EVP_MD_CTX_create();
395 		break;
396 
397 	case RTE_CRYPTO_AUTH_MD5_HMAC:
398 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
399 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
400 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
401 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
402 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
403 		sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
404 		sess->auth.hmac.ctx = EVP_MD_CTX_create();
405 		if (get_auth_algo(xform->auth.algo,
406 				&sess->auth.hmac.evp_algo) != 0)
407 			return -EINVAL;
408 		sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL,
409 				xform->auth.key.data, xform->auth.key.length);
410 		break;
411 
412 	default:
413 		return -EINVAL;
414 	}
415 
416 	sess->auth.aad_length = xform->auth.add_auth_data_length;
417 	sess->auth.digest_length = xform->auth.digest_length;
418 
419 	return 0;
420 }
421 
422 /* Set session AEAD parameters */
423 static int
424 openssl_set_session_aead_parameters(struct openssl_session *sess,
425 		const struct rte_crypto_sym_xform *xform)
426 {
427 	/* Select cipher direction */
428 	sess->cipher.direction = xform->cipher.op;
429 	/* Select cipher key */
430 	sess->cipher.key.length = xform->aead.key.length;
431 
432 	/* Set IV parameters */
433 	sess->iv.offset = xform->aead.iv.offset;
434 	sess->iv.length = xform->aead.iv.length;
435 
436 	/* Select auth generate/verify */
437 	sess->auth.operation = xform->auth.op;
438 	sess->auth.algo = xform->auth.algo;
439 
440 	/* Select auth algo */
441 	switch (xform->aead.algo) {
442 	case RTE_CRYPTO_AEAD_AES_GCM:
443 		sess->cipher.mode = OPENSSL_CIPHER_LIB;
444 		sess->aead_algo = xform->aead.algo;
445 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
446 
447 		if (get_aead_algo(sess->aead_algo, sess->cipher.key.length,
448 				&sess->cipher.evp_algo) != 0)
449 			return -EINVAL;
450 
451 		get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
452 			sess->cipher.key.data);
453 
454 		sess->chain_order = OPENSSL_CHAIN_COMBINED;
455 		break;
456 	default:
457 		return -EINVAL;
458 	}
459 
460 	sess->auth.aad_length = xform->aead.add_auth_data_length;
461 	sess->auth.digest_length = xform->aead.digest_length;
462 
463 	return 0;
464 }
465 
466 /** Parse crypto xform chain and set private session parameters */
467 int
468 openssl_set_session_parameters(struct openssl_session *sess,
469 		const struct rte_crypto_sym_xform *xform)
470 {
471 	const struct rte_crypto_sym_xform *cipher_xform = NULL;
472 	const struct rte_crypto_sym_xform *auth_xform = NULL;
473 	const struct rte_crypto_sym_xform *aead_xform = NULL;
474 
475 	sess->chain_order = openssl_get_chain_order(xform);
476 	switch (sess->chain_order) {
477 	case OPENSSL_CHAIN_ONLY_CIPHER:
478 		cipher_xform = xform;
479 		break;
480 	case OPENSSL_CHAIN_ONLY_AUTH:
481 		auth_xform = xform;
482 		break;
483 	case OPENSSL_CHAIN_CIPHER_AUTH:
484 		cipher_xform = xform;
485 		auth_xform = xform->next;
486 		break;
487 	case OPENSSL_CHAIN_AUTH_CIPHER:
488 		auth_xform = xform;
489 		cipher_xform = xform->next;
490 		break;
491 	case OPENSSL_CHAIN_COMBINED:
492 		aead_xform = xform;
493 		break;
494 	default:
495 		return -EINVAL;
496 	}
497 
498 	/* Default IV length = 0 */
499 	sess->iv.length = 0;
500 
501 	/* cipher_xform must be check before auth_xform */
502 	if (cipher_xform) {
503 		if (openssl_set_session_cipher_parameters(
504 				sess, cipher_xform)) {
505 			OPENSSL_LOG_ERR(
506 				"Invalid/unsupported cipher parameters");
507 			return -EINVAL;
508 		}
509 	}
510 
511 	if (auth_xform) {
512 		if (openssl_set_session_auth_parameters(sess, auth_xform)) {
513 			OPENSSL_LOG_ERR(
514 				"Invalid/unsupported auth parameters");
515 			return -EINVAL;
516 		}
517 	}
518 
519 	if (aead_xform) {
520 		if (openssl_set_session_aead_parameters(sess, aead_xform)) {
521 			OPENSSL_LOG_ERR(
522 				"Invalid/unsupported auth parameters");
523 			return -EINVAL;
524 		}
525 	}
526 
527 	return 0;
528 }
529 
530 /** Reset private session parameters */
531 void
532 openssl_reset_session(struct openssl_session *sess)
533 {
534 	EVP_CIPHER_CTX_free(sess->cipher.ctx);
535 
536 	if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
537 		EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
538 
539 	switch (sess->auth.mode) {
540 	case OPENSSL_AUTH_AS_AUTH:
541 		EVP_MD_CTX_destroy(sess->auth.auth.ctx);
542 		break;
543 	case OPENSSL_AUTH_AS_HMAC:
544 		EVP_PKEY_free(sess->auth.hmac.pkey);
545 		EVP_MD_CTX_destroy(sess->auth.hmac.ctx);
546 		break;
547 	default:
548 		break;
549 	}
550 }
551 
552 /** Provide session for operation */
553 static struct openssl_session *
554 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
555 {
556 	struct openssl_session *sess = NULL;
557 
558 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
559 		/* get existing session */
560 		if (likely(op->sym->session != NULL &&
561 				op->sym->session->dev_type ==
562 				RTE_CRYPTODEV_OPENSSL_PMD))
563 			sess = (struct openssl_session *)
564 				op->sym->session->_private;
565 	} else  {
566 		/* provide internal session */
567 		void *_sess = NULL;
568 
569 		if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) {
570 			sess = (struct openssl_session *)
571 				((struct rte_cryptodev_sym_session *)_sess)
572 				->_private;
573 
574 			if (unlikely(openssl_set_session_parameters(
575 					sess, op->sym->xform) != 0)) {
576 				rte_mempool_put(qp->sess_mp, _sess);
577 				sess = NULL;
578 			} else
579 				op->sym->session = _sess;
580 		}
581 	}
582 
583 	if (sess == NULL)
584 		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
585 
586 	return sess;
587 }
588 
589 /*
590  *------------------------------------------------------------------------------
591  * Process Operations
592  *------------------------------------------------------------------------------
593  */
594 static inline int
595 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
596 		uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
597 {
598 	struct rte_mbuf *m;
599 	int dstlen;
600 	int l, n = srclen;
601 	uint8_t *src;
602 
603 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
604 			m = m->next)
605 		offset -= rte_pktmbuf_data_len(m);
606 
607 	if (m == 0)
608 		return -1;
609 
610 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
611 
612 	l = rte_pktmbuf_data_len(m) - offset;
613 	if (srclen <= l) {
614 		if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
615 			return -1;
616 		*dst += l;
617 		return 0;
618 	}
619 
620 	if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
621 		return -1;
622 
623 	*dst += dstlen;
624 	n -= l;
625 
626 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
627 		src = rte_pktmbuf_mtod(m, uint8_t *);
628 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
629 		if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
630 			return -1;
631 		*dst += dstlen;
632 		n -= l;
633 	}
634 
635 	return 0;
636 }
637 
638 static inline int
639 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
640 		uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
641 {
642 	struct rte_mbuf *m;
643 	int dstlen;
644 	int l, n = srclen;
645 	uint8_t *src;
646 
647 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
648 			m = m->next)
649 		offset -= rte_pktmbuf_data_len(m);
650 
651 	if (m == 0)
652 		return -1;
653 
654 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
655 
656 	l = rte_pktmbuf_data_len(m) - offset;
657 	if (srclen <= l) {
658 		if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
659 			return -1;
660 		*dst += l;
661 		return 0;
662 	}
663 
664 	if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
665 		return -1;
666 
667 	*dst += dstlen;
668 	n -= l;
669 
670 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
671 		src = rte_pktmbuf_mtod(m, uint8_t *);
672 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
673 		if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
674 			return -1;
675 		*dst += dstlen;
676 		n -= l;
677 	}
678 
679 	return 0;
680 }
681 
682 /** Process standard openssl cipher encryption */
683 static int
684 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
685 		int offset, uint8_t *iv, uint8_t *key, int srclen,
686 		EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
687 {
688 	int totlen;
689 
690 	if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0)
691 		goto process_cipher_encrypt_err;
692 
693 	EVP_CIPHER_CTX_set_padding(ctx, 0);
694 
695 	if (process_openssl_encryption_update(mbuf_src, offset, &dst,
696 			srclen, ctx))
697 		goto process_cipher_encrypt_err;
698 
699 	if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
700 		goto process_cipher_encrypt_err;
701 
702 	return 0;
703 
704 process_cipher_encrypt_err:
705 	OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
706 	return -EINVAL;
707 }
708 
709 /** Process standard openssl cipher encryption */
710 static int
711 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
712 		uint8_t *iv, int srclen,
713 		EVP_CIPHER_CTX *ctx)
714 {
715 	uint8_t i;
716 	uint8_t encrypted_iv[DES_BLOCK_SIZE];
717 	int encrypted_ivlen;
718 
719 	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
720 			iv, DES_BLOCK_SIZE) <= 0)
721 		goto process_cipher_encrypt_err;
722 
723 	for (i = 0; i < srclen; i++)
724 		*(dst + i) = *(src + i) ^ (encrypted_iv[i]);
725 
726 	return 0;
727 
728 process_cipher_encrypt_err:
729 	OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
730 	return -EINVAL;
731 }
732 /** Process standard openssl cipher decryption */
733 static int
734 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
735 		int offset, uint8_t *iv, uint8_t *key, int srclen,
736 		EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
737 {
738 	int totlen;
739 
740 	if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0)
741 		goto process_cipher_decrypt_err;
742 
743 	EVP_CIPHER_CTX_set_padding(ctx, 0);
744 
745 	if (process_openssl_decryption_update(mbuf_src, offset, &dst,
746 			srclen, ctx))
747 		goto process_cipher_decrypt_err;
748 
749 	if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
750 		goto process_cipher_decrypt_err;
751 	return 0;
752 
753 process_cipher_decrypt_err:
754 	OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
755 	return -EINVAL;
756 }
757 
758 /** Process cipher des 3 ctr encryption, decryption algorithm */
759 static int
760 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
761 		int offset, uint8_t *iv, uint8_t *key, int srclen,
762 		EVP_CIPHER_CTX *ctx)
763 {
764 	uint8_t ebuf[8], ctr[8];
765 	int unused, n;
766 	struct rte_mbuf *m;
767 	uint8_t *src;
768 	int l;
769 
770 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
771 			m = m->next)
772 		offset -= rte_pktmbuf_data_len(m);
773 
774 	if (m == 0)
775 		goto process_cipher_des3ctr_err;
776 
777 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
778 	l = rte_pktmbuf_data_len(m) - offset;
779 
780 	/* We use 3DES encryption also for decryption.
781 	 * IV is not important for 3DES ecb
782 	 */
783 	if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
784 		goto process_cipher_des3ctr_err;
785 
786 	memcpy(ctr, iv, 8);
787 
788 	for (n = 0; n < srclen; n++) {
789 		if (n % 8 == 0) {
790 			if (EVP_EncryptUpdate(ctx,
791 					(unsigned char *)&ebuf, &unused,
792 					(const unsigned char *)&ctr, 8) <= 0)
793 				goto process_cipher_des3ctr_err;
794 			ctr_inc(ctr);
795 		}
796 		dst[n] = *(src++) ^ ebuf[n % 8];
797 
798 		l--;
799 		if (!l) {
800 			m = m->next;
801 			if (m) {
802 				src = rte_pktmbuf_mtod(m, uint8_t *);
803 				l = rte_pktmbuf_data_len(m);
804 			}
805 		}
806 	}
807 
808 	return 0;
809 
810 process_cipher_des3ctr_err:
811 	OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
812 	return -EINVAL;
813 }
814 
815 /** Process auth/encription aes-gcm algorithm */
816 static int
817 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
818 		int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen,
819 		uint8_t *key, uint8_t *dst, uint8_t *tag,
820 		EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
821 {
822 	int len = 0, unused = 0;
823 	uint8_t empty[] = {};
824 
825 	if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0)
826 		goto process_auth_encryption_gcm_err;
827 
828 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0)
829 		goto process_auth_encryption_gcm_err;
830 
831 	if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0)
832 		goto process_auth_encryption_gcm_err;
833 
834 	if (aadlen > 0)
835 		if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
836 			goto process_auth_encryption_gcm_err;
837 
838 	if (srclen > 0)
839 		if (process_openssl_encryption_update(mbuf_src, offset, &dst,
840 				srclen, ctx))
841 			goto process_auth_encryption_gcm_err;
842 
843 	/* Workaround open ssl bug in version less then 1.0.1f */
844 	if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
845 		goto process_auth_encryption_gcm_err;
846 
847 	if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
848 		goto process_auth_encryption_gcm_err;
849 
850 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
851 		goto process_auth_encryption_gcm_err;
852 
853 	return 0;
854 
855 process_auth_encryption_gcm_err:
856 	OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
857 	return -EINVAL;
858 }
859 
860 static int
861 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
862 		int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen,
863 		uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx,
864 		const EVP_CIPHER *algo)
865 {
866 	int len = 0, unused = 0;
867 	uint8_t empty[] = {};
868 
869 	if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0)
870 		goto process_auth_decryption_gcm_err;
871 
872 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0)
873 		goto process_auth_decryption_gcm_err;
874 
875 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
876 		goto process_auth_decryption_gcm_err;
877 
878 	if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0)
879 		goto process_auth_decryption_gcm_err;
880 
881 	if (aadlen > 0)
882 		if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
883 			goto process_auth_decryption_gcm_err;
884 
885 	if (srclen > 0)
886 		if (process_openssl_decryption_update(mbuf_src, offset, &dst,
887 				srclen, ctx))
888 			goto process_auth_decryption_gcm_err;
889 
890 	/* Workaround open ssl bug in version less then 1.0.1f */
891 	if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
892 		goto process_auth_decryption_gcm_err;
893 
894 	if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
895 		goto process_auth_decryption_gcm_final_err;
896 
897 	return 0;
898 
899 process_auth_decryption_gcm_err:
900 	OPENSSL_LOG_ERR("Process openssl auth description gcm failed");
901 	return -EINVAL;
902 
903 process_auth_decryption_gcm_final_err:
904 	return -EFAULT;
905 }
906 
907 /** Process standard openssl auth algorithms */
908 static int
909 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
910 		__rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
911 		int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
912 {
913 	size_t dstlen;
914 	struct rte_mbuf *m;
915 	int l, n = srclen;
916 	uint8_t *src;
917 
918 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
919 			m = m->next)
920 		offset -= rte_pktmbuf_data_len(m);
921 
922 	if (m == 0)
923 		goto process_auth_err;
924 
925 	if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
926 		goto process_auth_err;
927 
928 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
929 
930 	l = rte_pktmbuf_data_len(m) - offset;
931 	if (srclen <= l) {
932 		if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
933 			goto process_auth_err;
934 		goto process_auth_final;
935 	}
936 
937 	if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
938 		goto process_auth_err;
939 
940 	n -= l;
941 
942 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
943 		src = rte_pktmbuf_mtod(m, uint8_t *);
944 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
945 		if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
946 			goto process_auth_err;
947 		n -= l;
948 	}
949 
950 process_auth_final:
951 	if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
952 		goto process_auth_err;
953 	return 0;
954 
955 process_auth_err:
956 	OPENSSL_LOG_ERR("Process openssl auth failed");
957 	return -EINVAL;
958 }
959 
960 /** Process standard openssl auth algorithms with hmac */
961 static int
962 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
963 		__rte_unused uint8_t *iv, EVP_PKEY *pkey,
964 		int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
965 {
966 	size_t dstlen;
967 	struct rte_mbuf *m;
968 	int l, n = srclen;
969 	uint8_t *src;
970 
971 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
972 			m = m->next)
973 		offset -= rte_pktmbuf_data_len(m);
974 
975 	if (m == 0)
976 		goto process_auth_err;
977 
978 	if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0)
979 		goto process_auth_err;
980 
981 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
982 
983 	l = rte_pktmbuf_data_len(m) - offset;
984 	if (srclen <= l) {
985 		if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0)
986 			goto process_auth_err;
987 		goto process_auth_final;
988 	}
989 
990 	if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0)
991 		goto process_auth_err;
992 
993 	n -= l;
994 
995 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
996 		src = rte_pktmbuf_mtod(m, uint8_t *);
997 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
998 		if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0)
999 			goto process_auth_err;
1000 		n -= l;
1001 	}
1002 
1003 process_auth_final:
1004 	if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0)
1005 		goto process_auth_err;
1006 
1007 	return 0;
1008 
1009 process_auth_err:
1010 	OPENSSL_LOG_ERR("Process openssl auth failed");
1011 	return -EINVAL;
1012 }
1013 
1014 /*----------------------------------------------------------------------------*/
1015 
1016 /** Process auth/cipher combined operation */
1017 static void
1018 process_openssl_combined_op
1019 		(struct rte_crypto_op *op, struct openssl_session *sess,
1020 		struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1021 {
1022 	/* cipher */
1023 	uint8_t *dst = NULL, *iv, *tag, *aad;
1024 	int srclen, ivlen, aadlen, status = -1;
1025 	uint32_t offset;
1026 
1027 	/*
1028 	 * Segmented destination buffer is not supported for
1029 	 * encryption/decryption
1030 	 */
1031 	if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1032 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1033 		return;
1034 	}
1035 
1036 	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1037 			sess->iv.offset);
1038 	ivlen = sess->iv.length;
1039 	if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1040 		srclen = 0;
1041 		offset = op->sym->auth.data.offset;
1042 		aadlen = op->sym->auth.data.length;
1043 		aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1044 				op->sym->auth.data.offset);
1045 		tag = op->sym->auth.digest.data;
1046 		if (tag == NULL)
1047 			tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1048 				offset + aadlen);
1049 	} else {
1050 		srclen = op->sym->aead.data.length;
1051 		dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1052 				op->sym->aead.data.offset);
1053 		offset = op->sym->aead.data.offset;
1054 		aad = op->sym->aead.aad.data;
1055 		aadlen = sess->auth.aad_length;
1056 		tag = op->sym->aead.digest.data;
1057 		if (tag == NULL)
1058 			tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1059 				offset + srclen);
1060 	}
1061 
1062 	if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1063 		status = process_openssl_auth_encryption_gcm(
1064 				mbuf_src, offset, srclen,
1065 				aad, aadlen, iv, ivlen, sess->cipher.key.data,
1066 				dst, tag, sess->cipher.ctx,
1067 				sess->cipher.evp_algo);
1068 	else
1069 		status = process_openssl_auth_decryption_gcm(
1070 				mbuf_src, offset, srclen,
1071 				aad, aadlen, iv, ivlen, sess->cipher.key.data,
1072 				dst, tag, sess->cipher.ctx,
1073 				sess->cipher.evp_algo);
1074 
1075 	if (status != 0) {
1076 		if (status == (-EFAULT) &&
1077 				sess->auth.operation ==
1078 						RTE_CRYPTO_AUTH_OP_VERIFY)
1079 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1080 		else
1081 			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1082 	}
1083 }
1084 
1085 /** Process cipher operation */
1086 static void
1087 process_openssl_cipher_op
1088 		(struct rte_crypto_op *op, struct openssl_session *sess,
1089 		struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1090 {
1091 	uint8_t *dst, *iv;
1092 	int srclen, status;
1093 
1094 	/*
1095 	 * Segmented destination buffer is not supported for
1096 	 * encryption/decryption
1097 	 */
1098 	if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1099 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1100 		return;
1101 	}
1102 
1103 	srclen = op->sym->cipher.data.length;
1104 	dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1105 			op->sym->cipher.data.offset);
1106 
1107 	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1108 			sess->iv.offset);
1109 
1110 	if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1111 		if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1112 			status = process_openssl_cipher_encrypt(mbuf_src, dst,
1113 					op->sym->cipher.data.offset, iv,
1114 					sess->cipher.key.data, srclen,
1115 					sess->cipher.ctx,
1116 					sess->cipher.evp_algo);
1117 		else
1118 			status = process_openssl_cipher_decrypt(mbuf_src, dst,
1119 					op->sym->cipher.data.offset, iv,
1120 					sess->cipher.key.data, srclen,
1121 					sess->cipher.ctx,
1122 					sess->cipher.evp_algo);
1123 	else
1124 		status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1125 				op->sym->cipher.data.offset, iv,
1126 				sess->cipher.key.data, srclen,
1127 				sess->cipher.ctx);
1128 
1129 	if (status != 0)
1130 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1131 }
1132 
1133 /** Process cipher operation */
1134 static void
1135 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1136 		struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1137 		struct rte_mbuf *mbuf_dst)
1138 {
1139 	uint8_t *src, *dst, *iv;
1140 	uint8_t block_size, last_block_len;
1141 	int srclen, status = 0;
1142 
1143 	srclen = op->sym->cipher.data.length;
1144 	src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1145 			op->sym->cipher.data.offset);
1146 	dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1147 			op->sym->cipher.data.offset);
1148 
1149 	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1150 			sess->iv.offset);
1151 
1152 	block_size = DES_BLOCK_SIZE;
1153 
1154 	last_block_len = srclen % block_size;
1155 	if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1156 		/* Encrypt only with ECB mode XOR IV */
1157 		if (srclen < block_size) {
1158 			status = process_openssl_cipher_bpi_encrypt(src, dst,
1159 					iv, srclen,
1160 					sess->cipher.bpi_ctx);
1161 		} else {
1162 			srclen -= last_block_len;
1163 			/* Encrypt with the block aligned stream with CBC mode */
1164 			status = process_openssl_cipher_encrypt(mbuf_src, dst,
1165 					op->sym->cipher.data.offset, iv,
1166 					sess->cipher.key.data, srclen,
1167 					sess->cipher.ctx, sess->cipher.evp_algo);
1168 			if (last_block_len) {
1169 				/* Point at last block */
1170 				dst += srclen;
1171 				/*
1172 				 * IV is the last encrypted block from
1173 				 * the previous operation
1174 				 */
1175 				iv = dst - block_size;
1176 				src += srclen;
1177 				srclen = last_block_len;
1178 				/* Encrypt the last frame with ECB mode */
1179 				status |= process_openssl_cipher_bpi_encrypt(src,
1180 						dst, iv,
1181 						srclen, sess->cipher.bpi_ctx);
1182 			}
1183 		}
1184 	} else {
1185 		/* Decrypt only with ECB mode (encrypt, as it is same operation) */
1186 		if (srclen < block_size) {
1187 			status = process_openssl_cipher_bpi_encrypt(src, dst,
1188 					iv,
1189 					srclen,
1190 					sess->cipher.bpi_ctx);
1191 		} else {
1192 			if (last_block_len) {
1193 				/* Point at last block */
1194 				dst += srclen - last_block_len;
1195 				src += srclen - last_block_len;
1196 				/*
1197 				 * IV is the last full block
1198 				 */
1199 				iv = src - block_size;
1200 				/*
1201 				 * Decrypt the last frame with ECB mode
1202 				 * (encrypt, as it is the same operation)
1203 				 */
1204 				status = process_openssl_cipher_bpi_encrypt(src,
1205 						dst, iv,
1206 						last_block_len, sess->cipher.bpi_ctx);
1207 				/* Prepare parameters for CBC mode op */
1208 				iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1209 						sess->iv.offset);
1210 				dst += last_block_len - srclen;
1211 				srclen -= last_block_len;
1212 			}
1213 
1214 			/* Decrypt with CBC mode */
1215 			status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1216 					op->sym->cipher.data.offset, iv,
1217 					sess->cipher.key.data, srclen,
1218 					sess->cipher.ctx,
1219 					sess->cipher.evp_algo);
1220 		}
1221 	}
1222 
1223 	if (status != 0)
1224 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1225 }
1226 
1227 /** Process auth operation */
1228 static void
1229 process_openssl_auth_op
1230 		(struct rte_crypto_op *op, struct openssl_session *sess,
1231 		struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1232 {
1233 	uint8_t *dst;
1234 	int srclen, status;
1235 
1236 	srclen = op->sym->auth.data.length;
1237 
1238 	if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1239 		dst = (uint8_t *)rte_pktmbuf_append(mbuf_src,
1240 				sess->auth.digest_length);
1241 	else {
1242 		dst = op->sym->auth.digest.data;
1243 		if (dst == NULL)
1244 			dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1245 					op->sym->auth.data.offset +
1246 					op->sym->auth.data.length);
1247 	}
1248 
1249 	switch (sess->auth.mode) {
1250 	case OPENSSL_AUTH_AS_AUTH:
1251 		status = process_openssl_auth(mbuf_src, dst,
1252 				op->sym->auth.data.offset, NULL, NULL, srclen,
1253 				sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1254 		break;
1255 	case OPENSSL_AUTH_AS_HMAC:
1256 		status = process_openssl_auth_hmac(mbuf_src, dst,
1257 				op->sym->auth.data.offset, NULL,
1258 				sess->auth.hmac.pkey, srclen,
1259 				sess->auth.hmac.ctx, sess->auth.hmac.evp_algo);
1260 		break;
1261 	default:
1262 		status = -1;
1263 		break;
1264 	}
1265 
1266 	if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1267 		if (memcmp(dst, op->sym->auth.digest.data,
1268 				sess->auth.digest_length) != 0) {
1269 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1270 		}
1271 		/* Trim area used for digest from mbuf. */
1272 		rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length);
1273 	}
1274 
1275 	if (status != 0)
1276 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1277 }
1278 
1279 /** Process crypto operation for mbuf */
1280 static int
1281 process_op(const struct openssl_qp *qp, struct rte_crypto_op *op,
1282 		struct openssl_session *sess)
1283 {
1284 	struct rte_mbuf *msrc, *mdst;
1285 	int retval;
1286 
1287 	msrc = op->sym->m_src;
1288 	mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1289 
1290 	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1291 
1292 	switch (sess->chain_order) {
1293 	case OPENSSL_CHAIN_ONLY_CIPHER:
1294 		process_openssl_cipher_op(op, sess, msrc, mdst);
1295 		break;
1296 	case OPENSSL_CHAIN_ONLY_AUTH:
1297 		process_openssl_auth_op(op, sess, msrc, mdst);
1298 		break;
1299 	case OPENSSL_CHAIN_CIPHER_AUTH:
1300 		process_openssl_cipher_op(op, sess, msrc, mdst);
1301 		process_openssl_auth_op(op, sess, mdst, mdst);
1302 		break;
1303 	case OPENSSL_CHAIN_AUTH_CIPHER:
1304 		process_openssl_auth_op(op, sess, msrc, mdst);
1305 		process_openssl_cipher_op(op, sess, msrc, mdst);
1306 		break;
1307 	case OPENSSL_CHAIN_COMBINED:
1308 		process_openssl_combined_op(op, sess, msrc, mdst);
1309 		break;
1310 	case OPENSSL_CHAIN_CIPHER_BPI:
1311 		process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1312 		break;
1313 	default:
1314 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1315 		break;
1316 	}
1317 
1318 	/* Free session if a session-less crypto op */
1319 	if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1320 		openssl_reset_session(sess);
1321 		memset(sess, 0, sizeof(struct openssl_session));
1322 		rte_mempool_put(qp->sess_mp, op->sym->session);
1323 		op->sym->session = NULL;
1324 	}
1325 
1326 	if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1327 		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1328 
1329 	if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1330 		retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1331 	else
1332 		retval = -1;
1333 
1334 	return retval;
1335 }
1336 
1337 /*
1338  *------------------------------------------------------------------------------
1339  * PMD Framework
1340  *------------------------------------------------------------------------------
1341  */
1342 
1343 /** Enqueue burst */
1344 static uint16_t
1345 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1346 		uint16_t nb_ops)
1347 {
1348 	struct openssl_session *sess;
1349 	struct openssl_qp *qp = queue_pair;
1350 	int i, retval;
1351 
1352 	for (i = 0; i < nb_ops; i++) {
1353 		sess = get_session(qp, ops[i]);
1354 		if (unlikely(sess == NULL))
1355 			goto enqueue_err;
1356 
1357 		retval = process_op(qp, ops[i], sess);
1358 		if (unlikely(retval < 0))
1359 			goto enqueue_err;
1360 	}
1361 
1362 	qp->stats.enqueued_count += i;
1363 	return i;
1364 
1365 enqueue_err:
1366 	qp->stats.enqueue_err_count++;
1367 	return i;
1368 }
1369 
1370 /** Dequeue burst */
1371 static uint16_t
1372 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1373 		uint16_t nb_ops)
1374 {
1375 	struct openssl_qp *qp = queue_pair;
1376 
1377 	unsigned int nb_dequeued = 0;
1378 
1379 	nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1380 			(void **)ops, nb_ops, NULL);
1381 	qp->stats.dequeued_count += nb_dequeued;
1382 
1383 	return nb_dequeued;
1384 }
1385 
1386 /** Create OPENSSL crypto device */
1387 static int
1388 cryptodev_openssl_create(const char *name,
1389 			struct rte_vdev_device *vdev,
1390 			struct rte_crypto_vdev_init_params *init_params)
1391 {
1392 	struct rte_cryptodev *dev;
1393 	struct openssl_private *internals;
1394 
1395 	if (init_params->name[0] == '\0')
1396 		snprintf(init_params->name, sizeof(init_params->name),
1397 				"%s", name);
1398 
1399 	dev = rte_cryptodev_vdev_pmd_init(init_params->name,
1400 			sizeof(struct openssl_private),
1401 			init_params->socket_id,
1402 			vdev);
1403 	if (dev == NULL) {
1404 		OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1405 		goto init_error;
1406 	}
1407 
1408 	dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD;
1409 	dev->dev_ops = rte_openssl_pmd_ops;
1410 
1411 	/* register rx/tx burst functions for data path */
1412 	dev->dequeue_burst = openssl_pmd_dequeue_burst;
1413 	dev->enqueue_burst = openssl_pmd_enqueue_burst;
1414 
1415 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1416 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1417 			RTE_CRYPTODEV_FF_CPU_AESNI |
1418 			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1419 
1420 	/* Set vector instructions mode supported */
1421 	internals = dev->data->dev_private;
1422 
1423 	internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1424 	internals->max_nb_sessions = init_params->max_nb_sessions;
1425 
1426 	return 0;
1427 
1428 init_error:
1429 	OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1430 			init_params->name);
1431 
1432 	cryptodev_openssl_remove(vdev);
1433 	return -EFAULT;
1434 }
1435 
1436 /** Initialise OPENSSL crypto device */
1437 static int
1438 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1439 {
1440 	struct rte_crypto_vdev_init_params init_params = {
1441 		RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
1442 		RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
1443 		rte_socket_id(),
1444 		{0}
1445 	};
1446 	const char *name;
1447 	const char *input_args;
1448 
1449 	name = rte_vdev_device_name(vdev);
1450 	if (name == NULL)
1451 		return -EINVAL;
1452 	input_args = rte_vdev_device_args(vdev);
1453 
1454 	rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
1455 
1456 	RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
1457 			init_params.socket_id);
1458 	if (init_params.name[0] != '\0')
1459 		RTE_LOG(INFO, PMD, "  User defined name = %s\n",
1460 			init_params.name);
1461 	RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
1462 			init_params.max_nb_queue_pairs);
1463 	RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
1464 			init_params.max_nb_sessions);
1465 
1466 	return cryptodev_openssl_create(name, vdev, &init_params);
1467 }
1468 
1469 /** Uninitialise OPENSSL crypto device */
1470 static int
1471 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1472 {
1473 	const char *name;
1474 
1475 	name = rte_vdev_device_name(vdev);
1476 	if (name == NULL)
1477 		return -EINVAL;
1478 
1479 	RTE_LOG(INFO, PMD,
1480 		"Closing OPENSSL crypto device %s on numa socket %u\n",
1481 		name, rte_socket_id());
1482 
1483 	return 0;
1484 }
1485 
1486 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1487 	.probe = cryptodev_openssl_probe,
1488 	.remove = cryptodev_openssl_remove
1489 };
1490 
1491 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1492 	cryptodev_openssl_pmd_drv);
1493 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1494 	"max_nb_queue_pairs=<int> "
1495 	"max_nb_sessions=<int> "
1496 	"socket_id=<int>");
1497