xref: /dpdk/drivers/crypto/openssl/rte_openssl_pmd.c (revision 0e361f5a0ca96d4e228191ceaf800636e7beeea0)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2016-2017 Intel Corporation
3  */
4 
5 #include <rte_byteorder.h>
6 #include <rte_common.h>
7 #include <rte_hexdump.h>
8 #include <rte_cryptodev.h>
9 #include <cryptodev_pmd.h>
10 #include <bus_vdev_driver.h>
11 #include <rte_malloc.h>
12 #include <rte_cpuflags.h>
13 
14 #include <openssl/cmac.h>
15 #include <openssl/hmac.h>
16 #include <openssl/evp.h>
17 #include <openssl/ec.h>
18 
19 #include "openssl_pmd_private.h"
20 #include "compat.h"
21 
22 #define DES_BLOCK_SIZE 8
23 
24 static uint8_t cryptodev_driver_id;
25 
26 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
27 static HMAC_CTX *HMAC_CTX_new(void)
28 {
29 	HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
30 
31 	if (ctx != NULL)
32 		HMAC_CTX_init(ctx);
33 	return ctx;
34 }
35 
36 static void HMAC_CTX_free(HMAC_CTX *ctx)
37 {
38 	if (ctx != NULL) {
39 		HMAC_CTX_cleanup(ctx);
40 		OPENSSL_free(ctx);
41 	}
42 }
43 #endif
44 
45 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
46 
47 #include <openssl/provider.h>
48 #include <openssl/core_names.h>
49 #include <openssl/param_build.h>
50 
51 #define MAX_OSSL_ALGO_NAME_SIZE		16
52 
53 OSSL_PROVIDER *legacy;
54 OSSL_PROVIDER *deflt;
55 
56 static void ossl_legacy_provider_load(void)
57 {
58 	/* Load Multiple providers into the default (NULL) library context */
59 	legacy = OSSL_PROVIDER_load(NULL, "legacy");
60 	if (legacy == NULL) {
61 		OPENSSL_LOG(ERR, "Failed to load Legacy provider");
62 		return;
63 	}
64 
65 	deflt = OSSL_PROVIDER_load(NULL, "default");
66 	if (deflt == NULL) {
67 		OPENSSL_LOG(ERR, "Failed to load Default provider");
68 		OSSL_PROVIDER_unload(legacy);
69 		return;
70 	}
71 }
72 
73 static void ossl_legacy_provider_unload(void)
74 {
75 	OSSL_PROVIDER_unload(legacy);
76 	OSSL_PROVIDER_unload(deflt);
77 }
78 
79 static __rte_always_inline const char *
80 digest_name_get(enum rte_crypto_auth_algorithm algo)
81 {
82 	switch (algo) {
83 	case RTE_CRYPTO_AUTH_MD5_HMAC:
84 		return OSSL_DIGEST_NAME_MD5;
85 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
86 		return OSSL_DIGEST_NAME_SHA1;
87 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
88 		return OSSL_DIGEST_NAME_SHA2_224;
89 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
90 		return OSSL_DIGEST_NAME_SHA2_256;
91 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
92 		return OSSL_DIGEST_NAME_SHA2_384;
93 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
94 		return OSSL_DIGEST_NAME_SHA2_512;
95 	default:
96 		return NULL;
97 	}
98 }
99 #endif
100 
101 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
102 
103 /*
104  *------------------------------------------------------------------------------
105  * Session Prepare
106  *------------------------------------------------------------------------------
107  */
108 
109 /** Get xform chain order */
110 static enum openssl_chain_order
111 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
112 {
113 	enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
114 
115 	if (xform != NULL) {
116 		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
117 			if (xform->next == NULL)
118 				res =  OPENSSL_CHAIN_ONLY_AUTH;
119 			else if (xform->next->type ==
120 					RTE_CRYPTO_SYM_XFORM_CIPHER)
121 				res =  OPENSSL_CHAIN_AUTH_CIPHER;
122 		}
123 		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
124 			if (xform->next == NULL)
125 				res =  OPENSSL_CHAIN_ONLY_CIPHER;
126 			else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
127 				res =  OPENSSL_CHAIN_CIPHER_AUTH;
128 		}
129 		if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
130 			res = OPENSSL_CHAIN_COMBINED;
131 	}
132 
133 	return res;
134 }
135 
136 /** Get session cipher key from input cipher key */
137 static void
138 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key)
139 {
140 	memcpy(session_key, input_key, keylen);
141 }
142 
143 /** Get key ede 24 bytes standard from input key */
144 static int
145 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede)
146 {
147 	int res = 0;
148 
149 	/* Initialize keys - 24 bytes: [key1-key2-key3] */
150 	switch (keylen) {
151 	case 24:
152 		memcpy(key_ede, key, 24);
153 		break;
154 	case 16:
155 		/* K3 = K1 */
156 		memcpy(key_ede, key, 16);
157 		memcpy(key_ede + 16, key, 8);
158 		break;
159 	case 8:
160 		/* K1 = K2 = K3 (DES compatibility) */
161 		memcpy(key_ede, key, 8);
162 		memcpy(key_ede + 8, key, 8);
163 		memcpy(key_ede + 16, key, 8);
164 		break;
165 	default:
166 		OPENSSL_LOG(ERR, "Unsupported key size");
167 		res = -EINVAL;
168 	}
169 
170 	return res;
171 }
172 
173 /** Get adequate openssl function for input cipher algorithm */
174 static uint8_t
175 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
176 		const EVP_CIPHER **algo)
177 {
178 	int res = 0;
179 
180 	if (algo != NULL) {
181 		switch (sess_algo) {
182 		case RTE_CRYPTO_CIPHER_3DES_CBC:
183 			switch (keylen) {
184 			case 8:
185 				*algo = EVP_des_cbc();
186 				break;
187 			case 16:
188 				*algo = EVP_des_ede_cbc();
189 				break;
190 			case 24:
191 				*algo = EVP_des_ede3_cbc();
192 				break;
193 			default:
194 				res = -EINVAL;
195 			}
196 			break;
197 		case RTE_CRYPTO_CIPHER_3DES_CTR:
198 			break;
199 		case RTE_CRYPTO_CIPHER_AES_CBC:
200 			switch (keylen) {
201 			case 16:
202 				*algo = EVP_aes_128_cbc();
203 				break;
204 			case 24:
205 				*algo = EVP_aes_192_cbc();
206 				break;
207 			case 32:
208 				*algo = EVP_aes_256_cbc();
209 				break;
210 			default:
211 				res = -EINVAL;
212 			}
213 			break;
214 		case RTE_CRYPTO_CIPHER_AES_CTR:
215 			switch (keylen) {
216 			case 16:
217 				*algo = EVP_aes_128_ctr();
218 				break;
219 			case 24:
220 				*algo = EVP_aes_192_ctr();
221 				break;
222 			case 32:
223 				*algo = EVP_aes_256_ctr();
224 				break;
225 			default:
226 				res = -EINVAL;
227 			}
228 			break;
229 		default:
230 			res = -EINVAL;
231 			break;
232 		}
233 	} else {
234 		res = -EINVAL;
235 	}
236 
237 	return res;
238 }
239 
240 /** Get adequate openssl function for input auth algorithm */
241 static uint8_t
242 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
243 		const EVP_MD **algo)
244 {
245 	int res = 0;
246 
247 	if (algo != NULL) {
248 		switch (sessalgo) {
249 		case RTE_CRYPTO_AUTH_MD5:
250 		case RTE_CRYPTO_AUTH_MD5_HMAC:
251 			*algo = EVP_md5();
252 			break;
253 		case RTE_CRYPTO_AUTH_SHA1:
254 		case RTE_CRYPTO_AUTH_SHA1_HMAC:
255 			*algo = EVP_sha1();
256 			break;
257 		case RTE_CRYPTO_AUTH_SHA224:
258 		case RTE_CRYPTO_AUTH_SHA224_HMAC:
259 			*algo = EVP_sha224();
260 			break;
261 		case RTE_CRYPTO_AUTH_SHA256:
262 		case RTE_CRYPTO_AUTH_SHA256_HMAC:
263 			*algo = EVP_sha256();
264 			break;
265 		case RTE_CRYPTO_AUTH_SHA384:
266 		case RTE_CRYPTO_AUTH_SHA384_HMAC:
267 			*algo = EVP_sha384();
268 			break;
269 		case RTE_CRYPTO_AUTH_SHA512:
270 		case RTE_CRYPTO_AUTH_SHA512_HMAC:
271 			*algo = EVP_sha512();
272 			break;
273 		default:
274 			res = -EINVAL;
275 			break;
276 		}
277 	} else {
278 		res = -EINVAL;
279 	}
280 
281 	return res;
282 }
283 
284 /** Get adequate openssl function for input cipher algorithm */
285 static uint8_t
286 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
287 		const EVP_CIPHER **algo)
288 {
289 	int res = 0;
290 
291 	if (algo != NULL) {
292 		switch (sess_algo) {
293 		case RTE_CRYPTO_AEAD_AES_GCM:
294 			switch (keylen) {
295 			case 16:
296 				*algo = EVP_aes_128_gcm();
297 				break;
298 			case 24:
299 				*algo = EVP_aes_192_gcm();
300 				break;
301 			case 32:
302 				*algo = EVP_aes_256_gcm();
303 				break;
304 			default:
305 				res = -EINVAL;
306 			}
307 			break;
308 		case RTE_CRYPTO_AEAD_AES_CCM:
309 			switch (keylen) {
310 			case 16:
311 				*algo = EVP_aes_128_ccm();
312 				break;
313 			case 24:
314 				*algo = EVP_aes_192_ccm();
315 				break;
316 			case 32:
317 				*algo = EVP_aes_256_ccm();
318 				break;
319 			default:
320 				res = -EINVAL;
321 			}
322 			break;
323 		default:
324 			res = -EINVAL;
325 			break;
326 		}
327 	} else {
328 		res = -EINVAL;
329 	}
330 
331 	return res;
332 }
333 
334 /* Set session AEAD encryption parameters */
335 static int
336 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
337 		enum rte_crypto_aead_algorithm algo,
338 		uint8_t tag_len, const uint8_t *key,
339 		EVP_CIPHER_CTX **ctx)
340 {
341 	int iv_type = 0;
342 	unsigned int do_ccm;
343 
344 	sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
345 	sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
346 
347 	/* Select AEAD algo */
348 	switch (algo) {
349 	case RTE_CRYPTO_AEAD_AES_GCM:
350 		iv_type = EVP_CTRL_GCM_SET_IVLEN;
351 		if (tag_len != 16)
352 			return -EINVAL;
353 		do_ccm = 0;
354 		break;
355 	case RTE_CRYPTO_AEAD_AES_CCM:
356 		iv_type = EVP_CTRL_CCM_SET_IVLEN;
357 		/* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
358 		if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
359 			return -EINVAL;
360 		do_ccm = 1;
361 		break;
362 	default:
363 		return -ENOTSUP;
364 	}
365 
366 	sess->cipher.mode = OPENSSL_CIPHER_LIB;
367 	*ctx = EVP_CIPHER_CTX_new();
368 
369 	if (get_aead_algo(algo, sess->cipher.key.length,
370 			&sess->cipher.evp_algo) != 0)
371 		return -EINVAL;
372 
373 	get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
374 
375 	sess->chain_order = OPENSSL_CHAIN_COMBINED;
376 
377 	if (EVP_EncryptInit_ex(*ctx, sess->cipher.evp_algo,
378 			NULL, NULL, NULL) <= 0)
379 		return -EINVAL;
380 
381 	if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, sess->iv.length,
382 			NULL) <= 0)
383 		return -EINVAL;
384 
385 	if (do_ccm)
386 		EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG,
387 				tag_len, NULL);
388 
389 	if (EVP_EncryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0)
390 		return -EINVAL;
391 
392 	return 0;
393 }
394 
395 /* Set session AEAD decryption parameters */
396 static int
397 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
398 		enum rte_crypto_aead_algorithm algo,
399 		uint8_t tag_len, const uint8_t *key,
400 		EVP_CIPHER_CTX **ctx)
401 {
402 	int iv_type = 0;
403 	unsigned int do_ccm = 0;
404 
405 	sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
406 	sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
407 
408 	/* Select AEAD algo */
409 	switch (algo) {
410 	case RTE_CRYPTO_AEAD_AES_GCM:
411 		iv_type = EVP_CTRL_GCM_SET_IVLEN;
412 		if (tag_len != 16)
413 			return -EINVAL;
414 		break;
415 	case RTE_CRYPTO_AEAD_AES_CCM:
416 		iv_type = EVP_CTRL_CCM_SET_IVLEN;
417 		/* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
418 		if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
419 			return -EINVAL;
420 		do_ccm = 1;
421 		break;
422 	default:
423 		return -ENOTSUP;
424 	}
425 
426 	sess->cipher.mode = OPENSSL_CIPHER_LIB;
427 	*ctx = EVP_CIPHER_CTX_new();
428 
429 	if (get_aead_algo(algo, sess->cipher.key.length,
430 			&sess->cipher.evp_algo) != 0)
431 		return -EINVAL;
432 
433 	get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
434 
435 	sess->chain_order = OPENSSL_CHAIN_COMBINED;
436 
437 	if (EVP_DecryptInit_ex(*ctx, sess->cipher.evp_algo,
438 			NULL, NULL, NULL) <= 0)
439 		return -EINVAL;
440 
441 	if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type,
442 			sess->iv.length, NULL) <= 0)
443 		return -EINVAL;
444 
445 	if (do_ccm)
446 		EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG,
447 				tag_len, NULL);
448 
449 	if (EVP_DecryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0)
450 		return -EINVAL;
451 
452 	return 0;
453 }
454 
455 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30200000L)
456 static int openssl_aesni_ctx_clone(EVP_CIPHER_CTX **dest,
457 		struct openssl_session *sess)
458 {
459 	/* OpenSSL versions 3.0.0 <= V < 3.2.0 have no dupctx() implementation
460 	 * for AES-GCM and AES-CCM. In this case, we have to create new empty
461 	 * contexts and initialise, as we did the original context.
462 	 */
463 	if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC)
464 		sess->aead_algo = RTE_CRYPTO_AEAD_AES_GCM;
465 
466 	if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
467 		return openssl_set_sess_aead_enc_param(sess, sess->aead_algo,
468 				sess->auth.digest_length, sess->cipher.key.data,
469 				dest);
470 	else
471 		return openssl_set_sess_aead_dec_param(sess, sess->aead_algo,
472 				sess->auth.digest_length, sess->cipher.key.data,
473 				dest);
474 }
475 #endif
476 
477 /** Set session cipher parameters */
478 static int
479 openssl_set_session_cipher_parameters(struct openssl_session *sess,
480 		const struct rte_crypto_sym_xform *xform)
481 {
482 	/* Select cipher direction */
483 	sess->cipher.direction = xform->cipher.op;
484 	/* Select cipher key */
485 	sess->cipher.key.length = xform->cipher.key.length;
486 
487 	/* Set IV parameters */
488 	sess->iv.offset = xform->cipher.iv.offset;
489 	sess->iv.length = xform->cipher.iv.length;
490 
491 	/* Select cipher algo */
492 	switch (xform->cipher.algo) {
493 	case RTE_CRYPTO_CIPHER_3DES_CBC:
494 	case RTE_CRYPTO_CIPHER_AES_CBC:
495 	case RTE_CRYPTO_CIPHER_AES_CTR:
496 		sess->cipher.mode = OPENSSL_CIPHER_LIB;
497 		sess->cipher.algo = xform->cipher.algo;
498 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
499 
500 		if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
501 				&sess->cipher.evp_algo) != 0)
502 			return -EINVAL;
503 
504 		get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
505 			sess->cipher.key.data);
506 		if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
507 			if (EVP_EncryptInit_ex(sess->cipher.ctx,
508 					sess->cipher.evp_algo,
509 					NULL, xform->cipher.key.data,
510 					NULL) != 1) {
511 				return -EINVAL;
512 			}
513 		} else if (sess->cipher.direction ==
514 				RTE_CRYPTO_CIPHER_OP_DECRYPT) {
515 			if (EVP_DecryptInit_ex(sess->cipher.ctx,
516 					sess->cipher.evp_algo,
517 					NULL, xform->cipher.key.data,
518 					NULL) != 1) {
519 				return -EINVAL;
520 			}
521 		}
522 
523 		break;
524 
525 	case RTE_CRYPTO_CIPHER_3DES_CTR:
526 		sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
527 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
528 
529 		if (get_cipher_key_ede(xform->cipher.key.data,
530 				sess->cipher.key.length,
531 				sess->cipher.key.data) != 0)
532 			return -EINVAL;
533 
534 
535 		/* We use 3DES encryption also for decryption.
536 		 * IV is not important for 3DES ECB.
537 		 */
538 		if (EVP_EncryptInit_ex(sess->cipher.ctx, EVP_des_ede3_ecb(),
539 				NULL, sess->cipher.key.data,  NULL) != 1)
540 			return -EINVAL;
541 
542 		break;
543 
544 	case RTE_CRYPTO_CIPHER_DES_CBC:
545 		sess->cipher.algo = xform->cipher.algo;
546 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
547 		sess->cipher.evp_algo = EVP_des_cbc();
548 
549 		get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
550 			sess->cipher.key.data);
551 		if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
552 			if (EVP_EncryptInit_ex(sess->cipher.ctx,
553 					sess->cipher.evp_algo,
554 					NULL, xform->cipher.key.data,
555 					NULL) != 1) {
556 				return -EINVAL;
557 			}
558 		} else if (sess->cipher.direction ==
559 				RTE_CRYPTO_CIPHER_OP_DECRYPT) {
560 			if (EVP_DecryptInit_ex(sess->cipher.ctx,
561 					sess->cipher.evp_algo,
562 					NULL, xform->cipher.key.data,
563 					NULL) != 1) {
564 				return -EINVAL;
565 			}
566 		}
567 
568 		break;
569 
570 	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
571 		sess->cipher.algo = xform->cipher.algo;
572 		sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
573 		sess->cipher.ctx = EVP_CIPHER_CTX_new();
574 		sess->cipher.evp_algo = EVP_des_cbc();
575 
576 		sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
577 		/* IV will be ECB encrypted whether direction is encrypt or decrypt */
578 		if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
579 				NULL, xform->cipher.key.data, 0) != 1)
580 			return -EINVAL;
581 
582 		get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
583 			sess->cipher.key.data);
584 		if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
585 			if (EVP_EncryptInit_ex(sess->cipher.ctx,
586 					sess->cipher.evp_algo,
587 					NULL, xform->cipher.key.data,
588 					NULL) != 1) {
589 				return -EINVAL;
590 			}
591 		} else if (sess->cipher.direction ==
592 				RTE_CRYPTO_CIPHER_OP_DECRYPT) {
593 			if (EVP_DecryptInit_ex(sess->cipher.ctx,
594 					sess->cipher.evp_algo,
595 					NULL, xform->cipher.key.data,
596 					NULL) != 1) {
597 				return -EINVAL;
598 			}
599 		}
600 
601 		break;
602 	default:
603 		sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
604 		return -ENOTSUP;
605 	}
606 
607 	EVP_CIPHER_CTX_set_padding(sess->cipher.ctx, 0);
608 
609 	return 0;
610 }
611 
612 /* Set session auth parameters */
613 static int
614 openssl_set_session_auth_parameters(struct openssl_session *sess,
615 		const struct rte_crypto_sym_xform *xform)
616 {
617 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
618 	char algo_name[MAX_OSSL_ALGO_NAME_SIZE];
619 	OSSL_PARAM params[2];
620 	const char *algo;
621 	EVP_MAC *mac;
622 # endif
623 	/* Select auth generate/verify */
624 	sess->auth.operation = xform->auth.op;
625 	sess->auth.algo = xform->auth.algo;
626 
627 	sess->auth.digest_length = xform->auth.digest_length;
628 
629 	/* Select auth algo */
630 	switch (xform->auth.algo) {
631 	case RTE_CRYPTO_AUTH_AES_GMAC:
632 		/*
633 		 * OpenSSL requires GMAC to be a GCM operation
634 		 * with no cipher data length
635 		 */
636 		sess->cipher.key.length = xform->auth.key.length;
637 
638 		/* Set IV parameters */
639 		sess->iv.offset = xform->auth.iv.offset;
640 		sess->iv.length = xform->auth.iv.length;
641 
642 		if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
643 			return openssl_set_sess_aead_enc_param(sess,
644 						RTE_CRYPTO_AEAD_AES_GCM,
645 						xform->auth.digest_length,
646 						xform->auth.key.data,
647 						&sess->cipher.ctx);
648 		else
649 			return openssl_set_sess_aead_dec_param(sess,
650 						RTE_CRYPTO_AEAD_AES_GCM,
651 						xform->auth.digest_length,
652 						xform->auth.key.data,
653 						&sess->cipher.ctx);
654 		break;
655 
656 	case RTE_CRYPTO_AUTH_MD5:
657 	case RTE_CRYPTO_AUTH_SHA1:
658 	case RTE_CRYPTO_AUTH_SHA224:
659 	case RTE_CRYPTO_AUTH_SHA256:
660 	case RTE_CRYPTO_AUTH_SHA384:
661 	case RTE_CRYPTO_AUTH_SHA512:
662 		sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
663 		if (get_auth_algo(xform->auth.algo,
664 				&sess->auth.auth.evp_algo) != 0)
665 			return -EINVAL;
666 		sess->auth.auth.ctx = EVP_MD_CTX_create();
667 		break;
668 
669 	case RTE_CRYPTO_AUTH_AES_CMAC:
670 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
671 		if (xform->auth.key.length == 16)
672 			algo = SN_aes_128_cbc;
673 		else if (xform->auth.key.length == 24)
674 			algo = SN_aes_192_cbc;
675 		else if (xform->auth.key.length == 32)
676 			algo = SN_aes_256_cbc;
677 		else
678 			return -EINVAL;
679 
680 		strlcpy(algo_name, algo, sizeof(algo_name));
681 		params[0] = OSSL_PARAM_construct_utf8_string(
682 				OSSL_MAC_PARAM_CIPHER, algo_name, 0);
683 		params[1] = OSSL_PARAM_construct_end();
684 
685 		sess->auth.mode = OPENSSL_AUTH_AS_CMAC;
686 		mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL);
687 		sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac);
688 		EVP_MAC_free(mac);
689 
690 		if (EVP_MAC_init(sess->auth.cmac.ctx,
691 				xform->auth.key.data,
692 				xform->auth.key.length,
693 				params) != 1)
694 			return -EINVAL;
695 # else
696 		sess->auth.mode = OPENSSL_AUTH_AS_CMAC;
697 		sess->auth.cmac.ctx = CMAC_CTX_new();
698 		if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC,
699 				    xform->auth.key.length,
700 				    &sess->auth.cmac.evp_algo) != 0)
701 			return -EINVAL;
702 		if (CMAC_Init(sess->auth.cmac.ctx,
703 			      xform->auth.key.data,
704 			      xform->auth.key.length,
705 			      sess->auth.cmac.evp_algo, NULL) != 1)
706 			return -EINVAL;
707 # endif
708 		break;
709 
710 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
711 	case RTE_CRYPTO_AUTH_MD5_HMAC:
712 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
713 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
714 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
715 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
716 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
717 		sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
718 
719 		algo = digest_name_get(xform->auth.algo);
720 		if (!algo)
721 			return -EINVAL;
722 		strlcpy(algo_name, algo, sizeof(algo_name));
723 
724 		mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
725 		sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac);
726 		EVP_MAC_free(mac);
727 		if (get_auth_algo(xform->auth.algo,
728 				&sess->auth.hmac.evp_algo) != 0)
729 			return -EINVAL;
730 
731 		params[0] = OSSL_PARAM_construct_utf8_string("digest",
732 					algo_name, 0);
733 		params[1] = OSSL_PARAM_construct_end();
734 		if (EVP_MAC_init(sess->auth.hmac.ctx,
735 				xform->auth.key.data,
736 				xform->auth.key.length,
737 				params) != 1)
738 			return -EINVAL;
739 		break;
740 # else
741 	case RTE_CRYPTO_AUTH_MD5_HMAC:
742 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
743 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
744 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
745 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
746 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
747 		sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
748 		sess->auth.hmac.ctx = HMAC_CTX_new();
749 		if (get_auth_algo(xform->auth.algo,
750 				&sess->auth.hmac.evp_algo) != 0)
751 			return -EINVAL;
752 
753 		if (HMAC_Init_ex(sess->auth.hmac.ctx,
754 				xform->auth.key.data,
755 				xform->auth.key.length,
756 				sess->auth.hmac.evp_algo, NULL) != 1)
757 			return -EINVAL;
758 		break;
759 # endif
760 	default:
761 		return -ENOTSUP;
762 	}
763 
764 	return 0;
765 }
766 
767 /* Set session AEAD parameters */
768 static int
769 openssl_set_session_aead_parameters(struct openssl_session *sess,
770 		const struct rte_crypto_sym_xform *xform)
771 {
772 	/* Select cipher key */
773 	sess->cipher.key.length = xform->aead.key.length;
774 
775 	/* Set IV parameters */
776 	if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
777 		/*
778 		 * For AES-CCM, the actual IV is placed
779 		 * one byte after the start of the IV field,
780 		 * according to the API.
781 		 */
782 		sess->iv.offset = xform->aead.iv.offset + 1;
783 	else
784 		sess->iv.offset = xform->aead.iv.offset;
785 
786 	sess->iv.length = xform->aead.iv.length;
787 
788 	sess->auth.aad_length = xform->aead.aad_length;
789 	sess->auth.digest_length = xform->aead.digest_length;
790 
791 	sess->aead_algo = xform->aead.algo;
792 	/* Select cipher direction */
793 	if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
794 		return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
795 				xform->aead.digest_length, xform->aead.key.data,
796 				&sess->cipher.ctx);
797 	else
798 		return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
799 				xform->aead.digest_length, xform->aead.key.data,
800 				&sess->cipher.ctx);
801 }
802 
803 /** Parse crypto xform chain and set private session parameters */
804 int
805 openssl_set_session_parameters(struct openssl_session *sess,
806 		const struct rte_crypto_sym_xform *xform,
807 		uint16_t nb_queue_pairs)
808 {
809 	const struct rte_crypto_sym_xform *cipher_xform = NULL;
810 	const struct rte_crypto_sym_xform *auth_xform = NULL;
811 	const struct rte_crypto_sym_xform *aead_xform = NULL;
812 	int ret;
813 
814 	sess->chain_order = openssl_get_chain_order(xform);
815 	switch (sess->chain_order) {
816 	case OPENSSL_CHAIN_ONLY_CIPHER:
817 		cipher_xform = xform;
818 		break;
819 	case OPENSSL_CHAIN_ONLY_AUTH:
820 		auth_xform = xform;
821 		break;
822 	case OPENSSL_CHAIN_CIPHER_AUTH:
823 		cipher_xform = xform;
824 		auth_xform = xform->next;
825 		break;
826 	case OPENSSL_CHAIN_AUTH_CIPHER:
827 		auth_xform = xform;
828 		cipher_xform = xform->next;
829 		break;
830 	case OPENSSL_CHAIN_COMBINED:
831 		aead_xform = xform;
832 		break;
833 	default:
834 		return -EINVAL;
835 	}
836 
837 	/* Default IV length = 0 */
838 	sess->iv.length = 0;
839 
840 	/* cipher_xform must be check before auth_xform */
841 	if (cipher_xform) {
842 		ret = openssl_set_session_cipher_parameters(
843 				sess, cipher_xform);
844 		if (ret != 0) {
845 			OPENSSL_LOG(ERR,
846 				"Invalid/unsupported cipher parameters");
847 			return ret;
848 		}
849 	}
850 
851 	if (auth_xform) {
852 		ret = openssl_set_session_auth_parameters(sess, auth_xform);
853 		if (ret != 0) {
854 			OPENSSL_LOG(ERR,
855 				"Invalid/unsupported auth parameters");
856 			return ret;
857 		}
858 	}
859 
860 	if (aead_xform) {
861 		ret = openssl_set_session_aead_parameters(sess, aead_xform);
862 		if (ret != 0) {
863 			OPENSSL_LOG(ERR,
864 				"Invalid/unsupported AEAD parameters");
865 			return ret;
866 		}
867 	}
868 
869 	/*
870 	 * With only one queue pair, the array of copies is not needed.
871 	 * Otherwise, one entry per queue pair is required.
872 	 */
873 	sess->ctx_copies_len = nb_queue_pairs > 1 ? nb_queue_pairs : 0;
874 
875 	return 0;
876 }
877 
878 /** Reset private session parameters */
879 void
880 openssl_reset_session(struct openssl_session *sess)
881 {
882 	/* Free all the qp_ctx entries. */
883 	for (uint16_t i = 0; i < sess->ctx_copies_len; i++) {
884 		if (sess->qp_ctx[i].cipher != NULL) {
885 			EVP_CIPHER_CTX_free(sess->qp_ctx[i].cipher);
886 			sess->qp_ctx[i].cipher = NULL;
887 		}
888 
889 		switch (sess->auth.mode) {
890 		case OPENSSL_AUTH_AS_AUTH:
891 			EVP_MD_CTX_destroy(sess->qp_ctx[i].auth);
892 			sess->qp_ctx[i].auth = NULL;
893 			break;
894 		case OPENSSL_AUTH_AS_HMAC:
895 			free_hmac_ctx(sess->qp_ctx[i].hmac);
896 			sess->qp_ctx[i].hmac = NULL;
897 			break;
898 		case OPENSSL_AUTH_AS_CMAC:
899 			free_cmac_ctx(sess->qp_ctx[i].cmac);
900 			sess->qp_ctx[i].cmac = NULL;
901 			break;
902 		}
903 	}
904 
905 	EVP_CIPHER_CTX_free(sess->cipher.ctx);
906 
907 	switch (sess->auth.mode) {
908 	case OPENSSL_AUTH_AS_AUTH:
909 		EVP_MD_CTX_destroy(sess->auth.auth.ctx);
910 		break;
911 	case OPENSSL_AUTH_AS_HMAC:
912 		free_hmac_ctx(sess->auth.hmac.ctx);
913 		break;
914 	case OPENSSL_AUTH_AS_CMAC:
915 		free_cmac_ctx(sess->auth.cmac.ctx);
916 		break;
917 	}
918 
919 	if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
920 		EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
921 }
922 
923 /** Provide session for operation */
924 static void *
925 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
926 {
927 	struct openssl_session *sess = NULL;
928 	struct openssl_asym_session *asym_sess = NULL;
929 
930 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
931 		if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
932 			/* get existing session */
933 			if (likely(op->sym->session != NULL))
934 				sess = CRYPTODEV_GET_SYM_SESS_PRIV(
935 					op->sym->session);
936 		} else {
937 			if (likely(op->asym->session != NULL))
938 				asym_sess = (struct openssl_asym_session *)
939 						op->asym->session->sess_private_data;
940 			if (asym_sess == NULL)
941 				op->status =
942 					RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
943 			return asym_sess;
944 		}
945 	} else {
946 		struct rte_cryptodev_sym_session *_sess;
947 		/* sessionless asymmetric not supported */
948 		if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
949 			return NULL;
950 
951 		/* provide internal session */
952 		rte_mempool_get(qp->sess_mp, (void **)&_sess);
953 
954 		if (_sess == NULL)
955 			return NULL;
956 
957 		sess = (struct openssl_session *)_sess->driver_priv_data;
958 
959 		if (unlikely(openssl_set_session_parameters(sess,
960 				op->sym->xform, 1) != 0)) {
961 			rte_mempool_put(qp->sess_mp, _sess);
962 			sess = NULL;
963 		}
964 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
965 
966 	}
967 
968 	if (sess == NULL)
969 		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
970 
971 	return sess;
972 }
973 
974 /*
975  *------------------------------------------------------------------------------
976  * Process Operations
977  *------------------------------------------------------------------------------
978  */
979 static inline int
980 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
981 		uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
982 {
983 	struct rte_mbuf *m;
984 	int dstlen;
985 	int l, n = srclen;
986 	uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
987 
988 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
989 			m = m->next)
990 		offset -= rte_pktmbuf_data_len(m);
991 
992 	if (m == 0)
993 		return -1;
994 
995 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
996 	if (inplace)
997 		*dst = src;
998 
999 	l = rte_pktmbuf_data_len(m) - offset;
1000 	if (srclen <= l) {
1001 		if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
1002 			return -1;
1003 		*dst += l;
1004 		return 0;
1005 	}
1006 
1007 	if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1008 		return -1;
1009 
1010 	*dst += dstlen;
1011 	n -= l;
1012 
1013 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1014 		uint8_t diff = l - dstlen, rem;
1015 
1016 		src = rte_pktmbuf_mtod(m, uint8_t *);
1017 		l = RTE_MIN(rte_pktmbuf_data_len(m), n);
1018 		if (diff && inplace) {
1019 			rem = RTE_MIN(l,
1020 				(EVP_CIPHER_CTX_block_size(ctx) - diff));
1021 			if (EVP_EncryptUpdate(ctx, temp,
1022 						&dstlen, src, rem) <= 0)
1023 				return -1;
1024 			n -= rem;
1025 			rte_memcpy(*dst, temp, diff);
1026 			rte_memcpy(src, temp + diff, rem);
1027 			src += rem;
1028 			l -= rem;
1029 		}
1030 		if (inplace)
1031 			*dst = src;
1032 		if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1033 			return -1;
1034 		*dst += dstlen;
1035 		n -= l;
1036 	}
1037 
1038 	return 0;
1039 }
1040 
1041 static inline int
1042 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
1043 		uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
1044 {
1045 	struct rte_mbuf *m;
1046 	int dstlen;
1047 	int l, n = srclen;
1048 	uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
1049 
1050 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1051 			m = m->next)
1052 		offset -= rte_pktmbuf_data_len(m);
1053 
1054 	if (m == 0)
1055 		return -1;
1056 
1057 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1058 	if (inplace)
1059 		*dst = src;
1060 
1061 	l = rte_pktmbuf_data_len(m) - offset;
1062 	if (srclen <= l) {
1063 		if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
1064 			return -1;
1065 		*dst += l;
1066 		return 0;
1067 	}
1068 
1069 	if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1070 		return -1;
1071 
1072 	*dst += dstlen;
1073 	n -= l;
1074 
1075 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1076 		uint8_t diff = l - dstlen, rem;
1077 
1078 		src = rte_pktmbuf_mtod(m, uint8_t *);
1079 		l = RTE_MIN(rte_pktmbuf_data_len(m), n);
1080 		if (diff && inplace) {
1081 			rem = RTE_MIN(l,
1082 				(EVP_CIPHER_CTX_block_size(ctx) - diff));
1083 			if (EVP_DecryptUpdate(ctx, temp,
1084 						&dstlen, src, rem) <= 0)
1085 				return -1;
1086 			n -= rem;
1087 			rte_memcpy(*dst, temp, diff);
1088 			rte_memcpy(src, temp + diff, rem);
1089 			src += rem;
1090 			l -= rem;
1091 		}
1092 		if (inplace)
1093 			*dst = src;
1094 		if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1095 			return -1;
1096 		*dst += dstlen;
1097 		n -= l;
1098 	}
1099 
1100 	return 0;
1101 }
1102 
1103 /** Process standard openssl cipher encryption */
1104 static int
1105 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1106 		int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1107 		uint8_t inplace)
1108 {
1109 	int totlen;
1110 
1111 	if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1112 		goto process_cipher_encrypt_err;
1113 
1114 	if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1115 			srclen, ctx, inplace))
1116 		goto process_cipher_encrypt_err;
1117 
1118 	if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
1119 		goto process_cipher_encrypt_err;
1120 
1121 	return 0;
1122 
1123 process_cipher_encrypt_err:
1124 	OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
1125 	return -EINVAL;
1126 }
1127 
1128 /** Process standard openssl cipher encryption */
1129 static int
1130 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
1131 		uint8_t *iv, int srclen,
1132 		EVP_CIPHER_CTX *ctx)
1133 {
1134 	uint8_t i;
1135 	uint8_t encrypted_iv[DES_BLOCK_SIZE];
1136 	int encrypted_ivlen;
1137 
1138 	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
1139 			iv, DES_BLOCK_SIZE) <= 0)
1140 		goto process_cipher_encrypt_err;
1141 
1142 	for (i = 0; i < srclen; i++)
1143 		*(dst + i) = *(src + i) ^ (encrypted_iv[i]);
1144 
1145 	return 0;
1146 
1147 process_cipher_encrypt_err:
1148 	OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
1149 	return -EINVAL;
1150 }
1151 /** Process standard openssl cipher decryption */
1152 static int
1153 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1154 		int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1155 		uint8_t inplace)
1156 {
1157 	int totlen;
1158 
1159 	if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1160 		goto process_cipher_decrypt_err;
1161 
1162 	if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1163 			srclen, ctx, inplace))
1164 		goto process_cipher_decrypt_err;
1165 
1166 	if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
1167 		goto process_cipher_decrypt_err;
1168 	return 0;
1169 
1170 process_cipher_decrypt_err:
1171 	OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
1172 	return -EINVAL;
1173 }
1174 
1175 /** Process cipher des 3 ctr encryption, decryption algorithm */
1176 static int
1177 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
1178 		int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
1179 {
1180 	uint8_t ebuf[8];
1181 	uint64_t ctr;
1182 	int unused, n;
1183 	struct rte_mbuf *m;
1184 	uint8_t *src;
1185 	int l;
1186 
1187 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1188 			m = m->next)
1189 		offset -= rte_pktmbuf_data_len(m);
1190 
1191 	if (m == 0)
1192 		goto process_cipher_des3ctr_err;
1193 
1194 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1195 	l = rte_pktmbuf_data_len(m) - offset;
1196 
1197 	memcpy(&ctr, iv, 8);
1198 
1199 	for (n = 0; n < srclen; n++) {
1200 		if (n % 8 == 0) {
1201 			uint64_t cpu_ctr;
1202 
1203 			if (EVP_EncryptUpdate(ctx,
1204 					(unsigned char *)&ebuf, &unused,
1205 					(const unsigned char *)&ctr, 8) <= 0)
1206 				goto process_cipher_des3ctr_err;
1207 			cpu_ctr = rte_be_to_cpu_64(ctr);
1208 			cpu_ctr++;
1209 			ctr = rte_cpu_to_be_64(cpu_ctr);
1210 		}
1211 		dst[n] = *(src++) ^ ebuf[n % 8];
1212 
1213 		l--;
1214 		if (!l) {
1215 			m = m->next;
1216 			if (m) {
1217 				src = rte_pktmbuf_mtod(m, uint8_t *);
1218 				l = rte_pktmbuf_data_len(m);
1219 			}
1220 		}
1221 	}
1222 
1223 	return 0;
1224 
1225 process_cipher_des3ctr_err:
1226 	OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
1227 	return -EINVAL;
1228 }
1229 
1230 /** Process AES-GCM encrypt algorithm */
1231 static int
1232 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1233 		int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1234 		uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1235 {
1236 	int len = 0;
1237 #if OPENSSL_VERSION_NUMBER < 0x10100000L
1238 	int unused = 0;
1239 	uint8_t empty[] = {};
1240 #endif
1241 
1242 	if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1243 		goto process_auth_encryption_gcm_err;
1244 
1245 	if (aadlen > 0)
1246 		if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1247 			goto process_auth_encryption_gcm_err;
1248 
1249 	if (srclen > 0)
1250 		if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1251 				srclen, ctx, 0))
1252 			goto process_auth_encryption_gcm_err;
1253 
1254 #if OPENSSL_VERSION_NUMBER < 0x10100000L
1255 	/* Workaround open ssl bug in version less then 1.0.1f */
1256 	if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1257 		goto process_auth_encryption_gcm_err;
1258 #endif
1259 
1260 	if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1261 		goto process_auth_encryption_gcm_err;
1262 
1263 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1264 		goto process_auth_encryption_gcm_err;
1265 
1266 	return 0;
1267 
1268 process_auth_encryption_gcm_err:
1269 	OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
1270 	return -EINVAL;
1271 }
1272 
1273 /** Process AES-CCM encrypt algorithm */
1274 static int
1275 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1276 		int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1277 		uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1278 {
1279 	int len = 0;
1280 
1281 	if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1282 		goto process_auth_encryption_ccm_err;
1283 
1284 	if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1285 		goto process_auth_encryption_ccm_err;
1286 
1287 	if (aadlen > 0)
1288 		/*
1289 		 * For AES-CCM, the actual AAD is placed
1290 		 * 18 bytes after the start of the AAD field,
1291 		 * according to the API.
1292 		 */
1293 		if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1294 			goto process_auth_encryption_ccm_err;
1295 
1296 	if (srclen >= 0)
1297 		if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1298 				srclen, ctx, 0))
1299 			goto process_auth_encryption_ccm_err;
1300 
1301 	if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1302 		goto process_auth_encryption_ccm_err;
1303 
1304 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1305 		goto process_auth_encryption_ccm_err;
1306 
1307 	return 0;
1308 
1309 process_auth_encryption_ccm_err:
1310 	OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
1311 	return -EINVAL;
1312 }
1313 
1314 /** Process AES-GCM decrypt algorithm */
1315 static int
1316 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1317 		int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1318 		uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1319 {
1320 	int len = 0;
1321 #if OPENSSL_VERSION_NUMBER < 0x10100000L
1322 	int unused = 0;
1323 	uint8_t empty[] = {};
1324 #endif
1325 
1326 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1327 		goto process_auth_decryption_gcm_err;
1328 
1329 	if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1330 		goto process_auth_decryption_gcm_err;
1331 
1332 	if (aadlen > 0)
1333 		if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1334 			goto process_auth_decryption_gcm_err;
1335 
1336 	if (srclen > 0)
1337 		if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1338 				srclen, ctx, 0))
1339 			goto process_auth_decryption_gcm_err;
1340 
1341 #if OPENSSL_VERSION_NUMBER < 0x10100000L
1342 	/* Workaround open ssl bug in version less then 1.0.1f */
1343 	if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1344 		goto process_auth_decryption_gcm_err;
1345 #endif
1346 
1347 	if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1348 		return -EFAULT;
1349 
1350 	return 0;
1351 
1352 process_auth_decryption_gcm_err:
1353 	OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
1354 	return -EINVAL;
1355 }
1356 
1357 /** Process AES-CCM decrypt algorithm */
1358 static int
1359 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1360 		int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1361 		uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1362 		EVP_CIPHER_CTX *ctx)
1363 {
1364 	int len = 0;
1365 
1366 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1367 		goto process_auth_decryption_ccm_err;
1368 
1369 	if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1370 		goto process_auth_decryption_ccm_err;
1371 
1372 	if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1373 		goto process_auth_decryption_ccm_err;
1374 
1375 	if (aadlen > 0)
1376 		/*
1377 		 * For AES-CCM, the actual AAD is placed
1378 		 * 18 bytes after the start of the AAD field,
1379 		 * according to the API.
1380 		 */
1381 		if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1382 			goto process_auth_decryption_ccm_err;
1383 
1384 	if (srclen >= 0)
1385 		if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1386 				srclen, ctx, 0))
1387 			return -EFAULT;
1388 
1389 	return 0;
1390 
1391 process_auth_decryption_ccm_err:
1392 	OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
1393 	return -EINVAL;
1394 }
1395 
1396 /** Process standard openssl auth algorithms */
1397 static int
1398 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1399 		__rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1400 		int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1401 {
1402 	size_t dstlen;
1403 	struct rte_mbuf *m;
1404 	int l, n = srclen;
1405 	uint8_t *src;
1406 
1407 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1408 			m = m->next)
1409 		offset -= rte_pktmbuf_data_len(m);
1410 
1411 	if (m == 0)
1412 		goto process_auth_err;
1413 
1414 	if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1415 		goto process_auth_err;
1416 
1417 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1418 
1419 	l = rte_pktmbuf_data_len(m) - offset;
1420 	if (srclen <= l) {
1421 		if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1422 			goto process_auth_err;
1423 		goto process_auth_final;
1424 	}
1425 
1426 	if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1427 		goto process_auth_err;
1428 
1429 	n -= l;
1430 
1431 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1432 		src = rte_pktmbuf_mtod(m, uint8_t *);
1433 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1434 		if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1435 			goto process_auth_err;
1436 		n -= l;
1437 	}
1438 
1439 process_auth_final:
1440 	if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1441 		goto process_auth_err;
1442 	return 0;
1443 
1444 process_auth_err:
1445 	OPENSSL_LOG(ERR, "Process openssl auth failed");
1446 	return -EINVAL;
1447 }
1448 
1449 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1450 /** Process standard openssl auth algorithms with hmac/cmac */
1451 static int
1452 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1453 		int srclen, EVP_MAC_CTX *ctx)
1454 {
1455 	size_t dstlen;
1456 	struct rte_mbuf *m;
1457 	int l, n = srclen;
1458 	uint8_t *src;
1459 
1460 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1461 			m = m->next)
1462 		offset -= rte_pktmbuf_data_len(m);
1463 
1464 	if (m == 0)
1465 		goto process_auth_err;
1466 
1467 	if (EVP_MAC_init(ctx, NULL, 0, NULL) <= 0)
1468 		goto process_auth_err;
1469 
1470 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1471 
1472 	l = rte_pktmbuf_data_len(m) - offset;
1473 	if (srclen <= l) {
1474 		if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1)
1475 			goto process_auth_err;
1476 		goto process_auth_final;
1477 	}
1478 
1479 	if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1480 		goto process_auth_err;
1481 
1482 	n -= l;
1483 
1484 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1485 		src = rte_pktmbuf_mtod(m, uint8_t *);
1486 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1487 		if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1488 			goto process_auth_err;
1489 		n -= l;
1490 	}
1491 
1492 process_auth_final:
1493 	if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1)
1494 		goto process_auth_err;
1495 
1496 	return 0;
1497 
1498 process_auth_err:
1499 	OPENSSL_LOG(ERR, "Process openssl auth failed");
1500 	return -EINVAL;
1501 }
1502 # else
1503 /** Process standard openssl auth algorithms with hmac */
1504 static int
1505 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1506 		int srclen, HMAC_CTX *ctx)
1507 {
1508 	unsigned int dstlen;
1509 	struct rte_mbuf *m;
1510 	int l, n = srclen;
1511 	uint8_t *src;
1512 
1513 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1514 			m = m->next)
1515 		offset -= rte_pktmbuf_data_len(m);
1516 
1517 	if (m == 0)
1518 		goto process_auth_err;
1519 
1520 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1521 
1522 	l = rte_pktmbuf_data_len(m) - offset;
1523 	if (srclen <= l) {
1524 		if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1525 			goto process_auth_err;
1526 		goto process_auth_final;
1527 	}
1528 
1529 	if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1530 		goto process_auth_err;
1531 
1532 	n -= l;
1533 
1534 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1535 		src = rte_pktmbuf_mtod(m, uint8_t *);
1536 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1537 		if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1538 			goto process_auth_err;
1539 		n -= l;
1540 	}
1541 
1542 process_auth_final:
1543 	if (HMAC_Final(ctx, dst, &dstlen) != 1)
1544 		goto process_auth_err;
1545 
1546 	if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1547 		goto process_auth_err;
1548 
1549 	return 0;
1550 
1551 process_auth_err:
1552 	OPENSSL_LOG(ERR, "Process openssl auth failed");
1553 	return -EINVAL;
1554 }
1555 
1556 /** Process standard openssl auth algorithms with cmac */
1557 static int
1558 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1559 		int srclen, CMAC_CTX *ctx)
1560 {
1561 	unsigned int dstlen;
1562 	struct rte_mbuf *m;
1563 	int l, n = srclen;
1564 	uint8_t *src;
1565 
1566 	for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1567 			m = m->next)
1568 		offset -= rte_pktmbuf_data_len(m);
1569 
1570 	if (m == 0)
1571 		goto process_auth_err;
1572 
1573 	src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1574 
1575 	l = rte_pktmbuf_data_len(m) - offset;
1576 	if (srclen <= l) {
1577 		if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1578 			goto process_auth_err;
1579 		goto process_auth_final;
1580 	}
1581 
1582 	if (CMAC_Update(ctx, (unsigned char *)src, l) != 1)
1583 		goto process_auth_err;
1584 
1585 	n -= l;
1586 
1587 	for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1588 		src = rte_pktmbuf_mtod(m, uint8_t *);
1589 		l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1590 		if (CMAC_Update(ctx, (unsigned char *)src, l) != 1)
1591 			goto process_auth_err;
1592 		n -= l;
1593 	}
1594 
1595 process_auth_final:
1596 	if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1)
1597 		goto process_auth_err;
1598 
1599 	CMAC_CTX_cleanup(ctx);
1600 
1601 	return 0;
1602 
1603 process_auth_err:
1604 	OPENSSL_LOG(ERR, "Process openssl cmac auth failed");
1605 	return -EINVAL;
1606 }
1607 # endif
1608 /*----------------------------------------------------------------------------*/
1609 
1610 static inline EVP_CIPHER_CTX *
1611 get_local_cipher_ctx(struct openssl_session *sess, struct openssl_qp *qp)
1612 {
1613 	/* If the array is not being used, just return the main context. */
1614 	if (sess->ctx_copies_len == 0)
1615 		return sess->cipher.ctx;
1616 
1617 	EVP_CIPHER_CTX **lctx = &sess->qp_ctx[qp->id].cipher;
1618 
1619 	if (unlikely(*lctx == NULL)) {
1620 #if OPENSSL_VERSION_NUMBER >= 0x30200000L
1621 		/* EVP_CIPHER_CTX_dup() added in OSSL 3.2 */
1622 		*lctx = EVP_CIPHER_CTX_dup(sess->cipher.ctx);
1623 		return *lctx;
1624 #elif OPENSSL_VERSION_NUMBER >= 0x30000000L
1625 		if (sess->chain_order == OPENSSL_CHAIN_COMBINED) {
1626 			/* AESNI special-cased to use openssl_aesni_ctx_clone()
1627 			 * to allow for working around lack of
1628 			 * EVP_CIPHER_CTX_copy support for 3.0.0 <= OSSL Version
1629 			 * < 3.2.0.
1630 			 */
1631 			if (openssl_aesni_ctx_clone(lctx, sess) != 0)
1632 				*lctx = NULL;
1633 			return *lctx;
1634 		}
1635 #endif
1636 
1637 		*lctx = EVP_CIPHER_CTX_new();
1638 		EVP_CIPHER_CTX_copy(*lctx, sess->cipher.ctx);
1639 	}
1640 
1641 	return *lctx;
1642 }
1643 
1644 static inline EVP_MD_CTX *
1645 get_local_auth_ctx(struct openssl_session *sess, struct openssl_qp *qp)
1646 {
1647 	/* If the array is not being used, just return the main context. */
1648 	if (sess->ctx_copies_len == 0)
1649 		return sess->auth.auth.ctx;
1650 
1651 	EVP_MD_CTX **lctx = &sess->qp_ctx[qp->id].auth;
1652 
1653 	if (unlikely(*lctx == NULL)) {
1654 #if OPENSSL_VERSION_NUMBER >= 0x30100000L
1655 		/* EVP_MD_CTX_dup() added in OSSL 3.1 */
1656 		*lctx = EVP_MD_CTX_dup(sess->auth.auth.ctx);
1657 #else
1658 		*lctx = EVP_MD_CTX_new();
1659 		EVP_MD_CTX_copy(*lctx, sess->auth.auth.ctx);
1660 #endif
1661 	}
1662 
1663 	return *lctx;
1664 }
1665 
1666 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1667 static inline EVP_MAC_CTX *
1668 #else
1669 static inline HMAC_CTX *
1670 #endif
1671 get_local_hmac_ctx(struct openssl_session *sess, struct openssl_qp *qp)
1672 {
1673 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L)
1674 	/* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of
1675 	 * EVP_MAC_CTXs is broken, and doesn't actually reset their
1676 	 * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid
1677 	 * undefined behavior of provided macs on EVP_MAC
1678 	 * reinitialization"). In cases where the fix is not present,
1679 	 * fall back to duplicating the context every buffer as a
1680 	 * workaround, at the cost of performance.
1681 	 */
1682 	RTE_SET_USED(qp);
1683 	return EVP_MAC_CTX_dup(sess->auth.hmac.ctx);
1684 #else
1685 	if (sess->ctx_copies_len == 0)
1686 		return sess->auth.hmac.ctx;
1687 
1688 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1689 	EVP_MAC_CTX **lctx =
1690 #else
1691 	HMAC_CTX **lctx =
1692 #endif
1693 		&sess->qp_ctx[qp->id].hmac;
1694 
1695 	if (unlikely(*lctx == NULL)) {
1696 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1697 		*lctx = EVP_MAC_CTX_dup(sess->auth.hmac.ctx);
1698 #else
1699 		*lctx = HMAC_CTX_new();
1700 		HMAC_CTX_copy(*lctx, sess->auth.hmac.ctx);
1701 #endif
1702 	}
1703 
1704 	return *lctx;
1705 #endif
1706 }
1707 
1708 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1709 static inline EVP_MAC_CTX *
1710 #else
1711 static inline CMAC_CTX *
1712 #endif
1713 get_local_cmac_ctx(struct openssl_session *sess, struct openssl_qp *qp)
1714 {
1715 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L)
1716 	/* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of
1717 	 * EVP_MAC_CTXs is broken, and doesn't actually reset their
1718 	 * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid
1719 	 * undefined behavior of provided macs on EVP_MAC
1720 	 * reinitialization"). In cases where the fix is not present,
1721 	 * fall back to duplicating the context every buffer as a
1722 	 * workaround, at the cost of performance.
1723 	 */
1724 	RTE_SET_USED(qp);
1725 	return EVP_MAC_CTX_dup(sess->auth.cmac.ctx);
1726 #else
1727 	if (sess->ctx_copies_len == 0)
1728 		return sess->auth.cmac.ctx;
1729 
1730 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1731 	EVP_MAC_CTX **lctx =
1732 #else
1733 	CMAC_CTX **lctx =
1734 #endif
1735 		&sess->qp_ctx[qp->id].cmac;
1736 
1737 	if (unlikely(*lctx == NULL)) {
1738 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1739 		*lctx = EVP_MAC_CTX_dup(sess->auth.cmac.ctx);
1740 #else
1741 		*lctx = CMAC_CTX_new();
1742 		CMAC_CTX_copy(*lctx, sess->auth.cmac.ctx);
1743 #endif
1744 	}
1745 
1746 	return *lctx;
1747 #endif
1748 }
1749 
1750 /** Process auth/cipher combined operation */
1751 static void
1752 process_openssl_combined_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1753 		struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1754 		struct rte_mbuf *mbuf_dst)
1755 {
1756 	/* cipher */
1757 	uint8_t *dst = NULL, *iv, *tag, *aad;
1758 	int srclen, aadlen, status = -1;
1759 	uint32_t offset;
1760 	uint8_t taglen;
1761 
1762 	/*
1763 	 * Segmented destination buffer is not supported for
1764 	 * encryption/decryption
1765 	 */
1766 	if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1767 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1768 		return;
1769 	}
1770 
1771 	EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp);
1772 
1773 	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1774 			sess->iv.offset);
1775 	if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1776 		srclen = 0;
1777 		offset = op->sym->auth.data.offset;
1778 		aadlen = op->sym->auth.data.length;
1779 		aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1780 				op->sym->auth.data.offset);
1781 		tag = op->sym->auth.digest.data;
1782 		if (tag == NULL)
1783 			tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1784 				offset + aadlen);
1785 	} else {
1786 		srclen = op->sym->aead.data.length;
1787 		dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1788 				op->sym->aead.data.offset);
1789 		offset = op->sym->aead.data.offset;
1790 		aad = op->sym->aead.aad.data;
1791 		aadlen = sess->auth.aad_length;
1792 		tag = op->sym->aead.digest.data;
1793 		if (tag == NULL)
1794 			tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1795 				offset + srclen);
1796 	}
1797 
1798 	taglen = sess->auth.digest_length;
1799 
1800 	if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1801 		if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1802 				sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1803 			status = process_openssl_auth_encryption_gcm(
1804 					mbuf_src, offset, srclen,
1805 					aad, aadlen, iv,
1806 					dst, tag, ctx);
1807 		else
1808 			status = process_openssl_auth_encryption_ccm(
1809 					mbuf_src, offset, srclen,
1810 					aad, aadlen, iv,
1811 					dst, tag, taglen, ctx);
1812 
1813 	} else {
1814 		if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1815 				sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1816 			status = process_openssl_auth_decryption_gcm(
1817 					mbuf_src, offset, srclen,
1818 					aad, aadlen, iv,
1819 					dst, tag, ctx);
1820 		else
1821 			status = process_openssl_auth_decryption_ccm(
1822 					mbuf_src, offset, srclen,
1823 					aad, aadlen, iv,
1824 					dst, tag, taglen, ctx);
1825 	}
1826 
1827 	if (status != 0) {
1828 		if (status == (-EFAULT) &&
1829 				sess->auth.operation ==
1830 						RTE_CRYPTO_AUTH_OP_VERIFY)
1831 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1832 		else
1833 			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1834 	}
1835 }
1836 
1837 /** Process cipher operation */
1838 static void
1839 process_openssl_cipher_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1840 		struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1841 		struct rte_mbuf *mbuf_dst)
1842 {
1843 	uint8_t *dst, *iv;
1844 	int srclen, status;
1845 	uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0;
1846 
1847 	/*
1848 	 * Segmented OOP destination buffer is not supported for encryption/
1849 	 * decryption. In case of des3ctr, even inplace segmented buffers are
1850 	 * not supported.
1851 	 */
1852 	if (!rte_pktmbuf_is_contiguous(mbuf_dst) &&
1853 			(!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) {
1854 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1855 		return;
1856 	}
1857 
1858 	srclen = op->sym->cipher.data.length;
1859 	dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1860 			op->sym->cipher.data.offset);
1861 
1862 	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1863 			sess->iv.offset);
1864 
1865 	EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp);
1866 
1867 	if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1868 		if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1869 			status = process_openssl_cipher_encrypt(mbuf_src, dst,
1870 					op->sym->cipher.data.offset, iv,
1871 					srclen, ctx, inplace);
1872 		else
1873 			status = process_openssl_cipher_decrypt(mbuf_src, dst,
1874 					op->sym->cipher.data.offset, iv,
1875 					srclen, ctx, inplace);
1876 	else
1877 		status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1878 				op->sym->cipher.data.offset, iv, srclen, ctx);
1879 
1880 	if (status != 0)
1881 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1882 }
1883 
1884 /** Process cipher operation */
1885 static void
1886 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1887 		struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1888 		struct rte_mbuf *mbuf_dst)
1889 {
1890 	uint8_t *src, *dst, *iv;
1891 	uint8_t block_size, last_block_len;
1892 	int srclen, status = 0;
1893 
1894 	srclen = op->sym->cipher.data.length;
1895 	src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1896 			op->sym->cipher.data.offset);
1897 	dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1898 			op->sym->cipher.data.offset);
1899 
1900 	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1901 			sess->iv.offset);
1902 
1903 	block_size = DES_BLOCK_SIZE;
1904 
1905 	last_block_len = srclen % block_size;
1906 	if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1907 		/* Encrypt only with ECB mode XOR IV */
1908 		if (srclen < block_size) {
1909 			status = process_openssl_cipher_bpi_encrypt(src, dst,
1910 					iv, srclen,
1911 					sess->cipher.bpi_ctx);
1912 		} else {
1913 			srclen -= last_block_len;
1914 			/* Encrypt with the block aligned stream with CBC mode */
1915 			status = process_openssl_cipher_encrypt(mbuf_src, dst,
1916 					op->sym->cipher.data.offset, iv,
1917 					srclen, sess->cipher.ctx, 0);
1918 			if (last_block_len) {
1919 				/* Point at last block */
1920 				dst += srclen;
1921 				/*
1922 				 * IV is the last encrypted block from
1923 				 * the previous operation
1924 				 */
1925 				iv = dst - block_size;
1926 				src += srclen;
1927 				srclen = last_block_len;
1928 				/* Encrypt the last frame with ECB mode */
1929 				status |= process_openssl_cipher_bpi_encrypt(src,
1930 						dst, iv,
1931 						srclen, sess->cipher.bpi_ctx);
1932 			}
1933 		}
1934 	} else {
1935 		/* Decrypt only with ECB mode (encrypt, as it is same operation) */
1936 		if (srclen < block_size) {
1937 			status = process_openssl_cipher_bpi_encrypt(src, dst,
1938 					iv,
1939 					srclen,
1940 					sess->cipher.bpi_ctx);
1941 		} else {
1942 			if (last_block_len) {
1943 				/* Point at last block */
1944 				dst += srclen - last_block_len;
1945 				src += srclen - last_block_len;
1946 				/*
1947 				 * IV is the last full block
1948 				 */
1949 				iv = src - block_size;
1950 				/*
1951 				 * Decrypt the last frame with ECB mode
1952 				 * (encrypt, as it is the same operation)
1953 				 */
1954 				status = process_openssl_cipher_bpi_encrypt(src,
1955 						dst, iv,
1956 						last_block_len, sess->cipher.bpi_ctx);
1957 				/* Prepare parameters for CBC mode op */
1958 				iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1959 						sess->iv.offset);
1960 				dst += last_block_len - srclen;
1961 				srclen -= last_block_len;
1962 			}
1963 
1964 			/* Decrypt with CBC mode */
1965 			status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1966 					op->sym->cipher.data.offset, iv,
1967 					srclen, sess->cipher.ctx, 0);
1968 		}
1969 	}
1970 
1971 	if (status != 0)
1972 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1973 }
1974 
1975 /** Process auth operation */
1976 static void
1977 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1978 		struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1979 		struct rte_mbuf *mbuf_dst)
1980 {
1981 	uint8_t *dst;
1982 	int srclen, status;
1983 	EVP_MD_CTX *ctx_a;
1984 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1985 	EVP_MAC_CTX *ctx_h;
1986 	EVP_MAC_CTX *ctx_c;
1987 # else
1988 	HMAC_CTX *ctx_h;
1989 	CMAC_CTX *ctx_c;
1990 # endif
1991 
1992 	srclen = op->sym->auth.data.length;
1993 
1994 	dst = qp->temp_digest;
1995 
1996 	switch (sess->auth.mode) {
1997 	case OPENSSL_AUTH_AS_AUTH:
1998 		ctx_a = get_local_auth_ctx(sess, qp);
1999 		status = process_openssl_auth(mbuf_src, dst,
2000 				op->sym->auth.data.offset, NULL, NULL, srclen,
2001 				ctx_a, sess->auth.auth.evp_algo);
2002 		break;
2003 	case OPENSSL_AUTH_AS_HMAC:
2004 		ctx_h = get_local_hmac_ctx(sess, qp);
2005 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
2006 		status = process_openssl_auth_mac(mbuf_src, dst,
2007 				op->sym->auth.data.offset, srclen,
2008 				ctx_h);
2009 # else
2010 		status = process_openssl_auth_hmac(mbuf_src, dst,
2011 				op->sym->auth.data.offset, srclen,
2012 				ctx_h);
2013 # endif
2014 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L)
2015 		EVP_MAC_CTX_free(ctx_h);
2016 #endif
2017 		break;
2018 	case OPENSSL_AUTH_AS_CMAC:
2019 		ctx_c = get_local_cmac_ctx(sess, qp);
2020 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
2021 		status = process_openssl_auth_mac(mbuf_src, dst,
2022 				op->sym->auth.data.offset, srclen,
2023 				ctx_c);
2024 # else
2025 		status = process_openssl_auth_cmac(mbuf_src, dst,
2026 				op->sym->auth.data.offset, srclen,
2027 				ctx_c);
2028 # endif
2029 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L)
2030 		EVP_MAC_CTX_free(ctx_c);
2031 #endif
2032 		break;
2033 	default:
2034 		status = -1;
2035 		break;
2036 	}
2037 
2038 	if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
2039 		if (CRYPTO_memcmp(dst, op->sym->auth.digest.data,
2040 				sess->auth.digest_length) != 0) {
2041 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
2042 		}
2043 	} else {
2044 		uint8_t *auth_dst;
2045 
2046 		auth_dst = op->sym->auth.digest.data;
2047 		if (auth_dst == NULL)
2048 			auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
2049 					op->sym->auth.data.offset +
2050 					op->sym->auth.data.length);
2051 		memcpy(auth_dst, dst, sess->auth.digest_length);
2052 	}
2053 
2054 	if (status != 0)
2055 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
2056 }
2057 
2058 /* process dsa sign operation */
2059 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2060 static int
2061 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop,
2062 		struct openssl_asym_session *sess)
2063 {
2064 	struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
2065 	EVP_PKEY_CTX *dsa_ctx = NULL;
2066 	EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL);
2067 	EVP_PKEY *pkey = NULL;
2068 	OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld;
2069 	OSSL_PARAM *params = NULL;
2070 
2071 	size_t outlen;
2072 	unsigned char *dsa_sign_data;
2073 	const unsigned char *dsa_sign_data_p;
2074 	int ret = -1;
2075 
2076 	cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2077 	params = OSSL_PARAM_BLD_to_param(param_bld);
2078 	if (!params) {
2079 		OSSL_PARAM_BLD_free(param_bld);
2080 		return -1;
2081 	}
2082 
2083 	if (key_ctx == NULL
2084 		|| EVP_PKEY_fromdata_init(key_ctx) <= 0
2085 		|| EVP_PKEY_fromdata(key_ctx, &pkey,
2086 			EVP_PKEY_KEYPAIR, params) <= 0)
2087 		goto err_dsa_sign;
2088 
2089 	dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL);
2090 	if (!dsa_ctx)
2091 		goto err_dsa_sign;
2092 
2093 	if (EVP_PKEY_sign_init(dsa_ctx) <= 0)
2094 		goto err_dsa_sign;
2095 
2096 	if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data,
2097 						op->message.length) <= 0)
2098 		goto err_dsa_sign;
2099 
2100 	if (outlen <= 0)
2101 		goto err_dsa_sign;
2102 
2103 	dsa_sign_data = OPENSSL_malloc(outlen);
2104 	if (!dsa_sign_data)
2105 		goto err_dsa_sign;
2106 
2107 	if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data,
2108 						op->message.length) <= 0) {
2109 		OPENSSL_free(dsa_sign_data);
2110 		goto err_dsa_sign;
2111 	}
2112 
2113 	dsa_sign_data_p = (const unsigned char *)dsa_sign_data;
2114 	DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen);
2115 	if (!sign) {
2116 		OPENSSL_LOG(ERR, "%s:%d", __func__, __LINE__);
2117 		OPENSSL_free(dsa_sign_data);
2118 		goto err_dsa_sign;
2119 	} else {
2120 		const BIGNUM *r = NULL, *s = NULL;
2121 		get_dsa_sign(sign, &r, &s);
2122 
2123 		op->r.length = BN_bn2bin(r, op->r.data);
2124 		op->s.length = BN_bn2bin(s, op->s.data);
2125 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2126 	}
2127 
2128 	ret = 0;
2129 	DSA_SIG_free(sign);
2130 	OPENSSL_free(dsa_sign_data);
2131 
2132 err_dsa_sign:
2133 	if (params)
2134 		OSSL_PARAM_free(params);
2135 	EVP_PKEY_CTX_free(key_ctx);
2136 	EVP_PKEY_CTX_free(dsa_ctx);
2137 	EVP_PKEY_free(pkey);
2138 	return ret;
2139 }
2140 
2141 /* process dsa verify operation */
2142 static int
2143 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop,
2144 		struct openssl_asym_session *sess)
2145 {
2146 	struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
2147 	DSA_SIG *sign = DSA_SIG_new();
2148 	BIGNUM *r = NULL, *s = NULL;
2149 	BIGNUM *pub_key = NULL;
2150 	OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld;
2151 	OSSL_PARAM *params = NULL;
2152 	EVP_PKEY *pkey = NULL;
2153 	EVP_PKEY_CTX *dsa_ctx = NULL;
2154 	EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL);
2155 	unsigned char *dsa_sig = NULL;
2156 	size_t sig_len;
2157 	int ret = -1;
2158 
2159 	cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2160 	if (!param_bld) {
2161 		OPENSSL_LOG(ERR, " %s:%d", __func__, __LINE__);
2162 		return -1;
2163 	}
2164 
2165 	r = BN_bin2bn(op->r.data, op->r.length, r);
2166 	s = BN_bin2bn(op->s.data, op->s.length,	s);
2167 	pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key);
2168 	if (!r || !s || !pub_key) {
2169 		BN_free(r);
2170 		BN_free(s);
2171 		BN_free(pub_key);
2172 		OSSL_PARAM_BLD_free(param_bld);
2173 		goto err_dsa_verify;
2174 	}
2175 
2176 	set_dsa_sign(sign, r, s);
2177 	if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) {
2178 		OSSL_PARAM_BLD_free(param_bld);
2179 		goto err_dsa_verify;
2180 	}
2181 
2182 	params = OSSL_PARAM_BLD_to_param(param_bld);
2183 	if (!params) {
2184 		OSSL_PARAM_BLD_free(param_bld);
2185 		goto err_dsa_verify;
2186 	}
2187 
2188 	if (key_ctx == NULL
2189 		|| EVP_PKEY_fromdata_init(key_ctx) <= 0
2190 		|| EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
2191 		goto err_dsa_verify;
2192 
2193 	dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL);
2194 	if (!dsa_ctx)
2195 		goto err_dsa_verify;
2196 
2197 	if (!sign)
2198 		goto err_dsa_verify;
2199 
2200 	sig_len = i2d_DSA_SIG(sign, &dsa_sig);
2201 	if (EVP_PKEY_verify_init(dsa_ctx) <= 0)
2202 		goto err_dsa_verify;
2203 
2204 	ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len,
2205 					op->message.data, op->message.length);
2206 	if (ret == 1) {
2207 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2208 		ret = 0;
2209 	}
2210 
2211 	OPENSSL_free(dsa_sig);
2212 err_dsa_verify:
2213 	if (sign)
2214 		DSA_SIG_free(sign);
2215 	if (params)
2216 		OSSL_PARAM_free(params);
2217 	EVP_PKEY_CTX_free(key_ctx);
2218 	EVP_PKEY_CTX_free(dsa_ctx);
2219 
2220 	BN_free(pub_key);
2221 	EVP_PKEY_free(pkey);
2222 
2223 	return ret;
2224 }
2225 #else
2226 static int
2227 process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
2228 		struct openssl_asym_session *sess)
2229 {
2230 	struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
2231 	DSA *dsa = sess->u.s.dsa;
2232 	DSA_SIG *sign = NULL;
2233 
2234 	sign = DSA_do_sign(op->message.data,
2235 			op->message.length,
2236 			dsa);
2237 
2238 	if (sign == NULL) {
2239 		OPENSSL_LOG(ERR, "%s:%d", __func__, __LINE__);
2240 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2241 	} else {
2242 		const BIGNUM *r = NULL, *s = NULL;
2243 		get_dsa_sign(sign, &r, &s);
2244 
2245 		op->r.length = BN_bn2bin(r, op->r.data);
2246 		op->s.length = BN_bn2bin(s, op->s.data);
2247 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2248 	}
2249 
2250 	DSA_SIG_free(sign);
2251 
2252 	return 0;
2253 }
2254 
2255 /* process dsa verify operation */
2256 static int
2257 process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
2258 		struct openssl_asym_session *sess)
2259 {
2260 	struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
2261 	DSA *dsa = sess->u.s.dsa;
2262 	int ret;
2263 	DSA_SIG *sign = DSA_SIG_new();
2264 	BIGNUM *r = NULL, *s = NULL;
2265 	BIGNUM *pub_key = NULL;
2266 
2267 	if (sign == NULL) {
2268 		OPENSSL_LOG(ERR, " %s:%d", __func__, __LINE__);
2269 		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2270 		return -1;
2271 	}
2272 
2273 	r = BN_bin2bn(op->r.data,
2274 			op->r.length,
2275 			r);
2276 	s = BN_bin2bn(op->s.data,
2277 			op->s.length,
2278 			s);
2279 	pub_key = BN_bin2bn(op->y.data,
2280 			op->y.length,
2281 			pub_key);
2282 	if (!r || !s || !pub_key) {
2283 		BN_free(r);
2284 		BN_free(s);
2285 		BN_free(pub_key);
2286 
2287 		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2288 		return -1;
2289 	}
2290 	set_dsa_sign(sign, r, s);
2291 	set_dsa_pub_key(dsa, pub_key);
2292 
2293 	ret = DSA_do_verify(op->message.data,
2294 			op->message.length,
2295 			sign,
2296 			dsa);
2297 
2298 	if (ret != 1)
2299 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2300 	else
2301 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2302 
2303 	DSA_SIG_free(sign);
2304 
2305 	return 0;
2306 }
2307 #endif
2308 
2309 /* process dh operation */
2310 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2311 static int
2312 process_openssl_dh_op_evp(struct rte_crypto_op *cop,
2313 		struct openssl_asym_session *sess)
2314 {
2315 	struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2316 	OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld;
2317 	OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer;
2318 	OSSL_PARAM *params = NULL;
2319 	EVP_PKEY *dhpkey = NULL;
2320 	EVP_PKEY *peerkey = NULL;
2321 	BIGNUM *priv_key = NULL;
2322 	BIGNUM *pub_key = NULL;
2323 	int ret = -1;
2324 
2325 	cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2326 	EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
2327 	if (dh_ctx == NULL || param_bld == NULL)
2328 		return ret;
2329 
2330 	if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2331 		OSSL_PARAM *params_peer = NULL;
2332 
2333 		if (!param_bld_peer)
2334 			return ret;
2335 
2336 		pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length,
2337 					pub_key);
2338 		if (pub_key == NULL) {
2339 			OSSL_PARAM_BLD_free(param_bld_peer);
2340 			return ret;
2341 		}
2342 
2343 		if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY,
2344 				pub_key)) {
2345 			OPENSSL_LOG(ERR, "Failed to set public key");
2346 			OSSL_PARAM_BLD_free(param_bld_peer);
2347 			BN_free(pub_key);
2348 			return ret;
2349 		}
2350 
2351 		params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer);
2352 		if (!params_peer) {
2353 			OSSL_PARAM_BLD_free(param_bld_peer);
2354 			BN_free(pub_key);
2355 			return ret;
2356 		}
2357 
2358 		EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
2359 		if (EVP_PKEY_keygen_init(peer_ctx) != 1) {
2360 			OSSL_PARAM_free(params_peer);
2361 			BN_free(pub_key);
2362 			return ret;
2363 		}
2364 
2365 		if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) {
2366 			EVP_PKEY_CTX_free(peer_ctx);
2367 			OSSL_PARAM_free(params_peer);
2368 			BN_free(pub_key);
2369 			return ret;
2370 		}
2371 
2372 		if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) {
2373 			EVP_PKEY_CTX_free(peer_ctx);
2374 			OSSL_PARAM_free(params_peer);
2375 			BN_free(pub_key);
2376 			return ret;
2377 		}
2378 
2379 		priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length,
2380 					priv_key);
2381 		if (priv_key == NULL) {
2382 			EVP_PKEY_CTX_free(peer_ctx);
2383 			OSSL_PARAM_free(params_peer);
2384 			BN_free(pub_key);
2385 			return ret;
2386 		}
2387 
2388 		if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY,
2389 				priv_key)) {
2390 			OPENSSL_LOG(ERR, "Failed to set private key");
2391 			EVP_PKEY_CTX_free(peer_ctx);
2392 			OSSL_PARAM_free(params_peer);
2393 			BN_free(pub_key);
2394 			BN_free(priv_key);
2395 			return ret;
2396 		}
2397 
2398 		OSSL_PARAM_free(params_peer);
2399 		EVP_PKEY_CTX_free(peer_ctx);
2400 	}
2401 
2402 	params = OSSL_PARAM_BLD_to_param(param_bld);
2403 	if (!params)
2404 		goto err_dh;
2405 
2406 	if (EVP_PKEY_keygen_init(dh_ctx) != 1)
2407 		goto err_dh;
2408 
2409 	if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1)
2410 		goto err_dh;
2411 
2412 	if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1)
2413 		goto err_dh;
2414 
2415 	if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2416 		OPENSSL_LOG(DEBUG, "%s:%d updated pub key", __func__, __LINE__);
2417 		if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key))
2418 			goto err_dh;
2419 				/* output public key */
2420 		op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data);
2421 	}
2422 
2423 	if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2424 
2425 		OPENSSL_LOG(DEBUG, "%s:%d updated priv key", __func__, __LINE__);
2426 		if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key))
2427 			goto err_dh;
2428 
2429 		/* provide generated private key back to user */
2430 		op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data);
2431 	}
2432 
2433 	if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2434 		size_t skey_len;
2435 		EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL);
2436 		if (!sc_ctx)
2437 			goto err_dh;
2438 
2439 		if (EVP_PKEY_derive_init(sc_ctx) <= 0) {
2440 			EVP_PKEY_CTX_free(sc_ctx);
2441 			goto err_dh;
2442 		}
2443 
2444 		if (!peerkey) {
2445 			EVP_PKEY_CTX_free(sc_ctx);
2446 			goto err_dh;
2447 		}
2448 
2449 		if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) {
2450 			EVP_PKEY_CTX_free(sc_ctx);
2451 			goto err_dh;
2452 		}
2453 
2454 		/* Determine buffer length */
2455 		if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) {
2456 			EVP_PKEY_CTX_free(sc_ctx);
2457 			goto err_dh;
2458 		}
2459 
2460 		if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) {
2461 			EVP_PKEY_CTX_free(sc_ctx);
2462 			goto err_dh;
2463 		}
2464 
2465 		op->shared_secret.length = skey_len;
2466 		EVP_PKEY_CTX_free(sc_ctx);
2467 	}
2468 
2469 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2470 	ret = 0;
2471 
2472  err_dh:
2473 	BN_free(pub_key);
2474 	BN_free(priv_key);
2475 	if (params)
2476 		OSSL_PARAM_free(params);
2477 	EVP_PKEY_free(dhpkey);
2478 	EVP_PKEY_free(peerkey);
2479 
2480 	EVP_PKEY_CTX_free(dh_ctx);
2481 
2482 	return ret;
2483 }
2484 #else
2485 static int
2486 process_openssl_dh_op(struct rte_crypto_op *cop,
2487 		struct openssl_asym_session *sess)
2488 {
2489 	struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2490 	struct rte_crypto_asym_op *asym_op = cop->asym;
2491 	DH *dh_key = sess->u.dh.dh_key;
2492 	BIGNUM *priv_key = NULL;
2493 	int ret = 0;
2494 
2495 	if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2496 		/* compute shared secret using peer public key
2497 		 * and current private key
2498 		 * shared secret = peer_key ^ priv_key mod p
2499 		 */
2500 		BIGNUM *peer_key = NULL;
2501 
2502 		/* copy private key and peer key and compute shared secret */
2503 		peer_key = BN_bin2bn(op->pub_key.data,
2504 				op->pub_key.length,
2505 				peer_key);
2506 		if (peer_key == NULL) {
2507 			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2508 			return -1;
2509 		}
2510 		priv_key = BN_bin2bn(op->priv_key.data,
2511 				op->priv_key.length,
2512 				priv_key);
2513 		if (priv_key == NULL) {
2514 			BN_free(peer_key);
2515 			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2516 			return -1;
2517 		}
2518 		ret = set_dh_priv_key(dh_key, priv_key);
2519 		if (ret) {
2520 			OPENSSL_LOG(ERR, "Failed to set private key");
2521 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2522 			BN_free(peer_key);
2523 			BN_free(priv_key);
2524 			return 0;
2525 		}
2526 
2527 		ret = DH_compute_key(
2528 				op->shared_secret.data,
2529 				peer_key, dh_key);
2530 		if (ret < 0) {
2531 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2532 			BN_free(peer_key);
2533 			/* priv key is already loaded into dh,
2534 			 * let's not free that directly here.
2535 			 * DH_free() will auto free it later.
2536 			 */
2537 			return 0;
2538 		}
2539 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2540 		op->shared_secret.length = ret;
2541 		BN_free(peer_key);
2542 		return 0;
2543 	}
2544 
2545 	/*
2546 	 * other options are public and private key generations.
2547 	 *
2548 	 * if user provides private key,
2549 	 * then first set DH with user provided private key
2550 	 */
2551 	if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE &&
2552 			op->priv_key.length) {
2553 		/* generate public key using user-provided private key
2554 		 * pub_key = g ^ priv_key mod p
2555 		 */
2556 
2557 		/* load private key into DH */
2558 		priv_key = BN_bin2bn(op->priv_key.data,
2559 				op->priv_key.length,
2560 				priv_key);
2561 		if (priv_key == NULL) {
2562 			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2563 			return -1;
2564 		}
2565 		ret = set_dh_priv_key(dh_key, priv_key);
2566 		if (ret) {
2567 			OPENSSL_LOG(ERR, "Failed to set private key");
2568 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2569 			BN_free(priv_key);
2570 			return 0;
2571 		}
2572 	}
2573 
2574 	/* generate public and private key pair.
2575 	 *
2576 	 * if private key already set, generates only public key.
2577 	 *
2578 	 * if private key is not already set, then set it to random value
2579 	 * and update internal private key.
2580 	 */
2581 	if (!DH_generate_key(dh_key)) {
2582 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2583 		return 0;
2584 	}
2585 
2586 	if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2587 		const BIGNUM *pub_key = NULL;
2588 
2589 		OPENSSL_LOG(DEBUG, "%s:%d update public key",
2590 				__func__, __LINE__);
2591 
2592 		/* get the generated keys */
2593 		get_dh_pub_key(dh_key, &pub_key);
2594 
2595 		/* output public key */
2596 		op->pub_key.length = BN_bn2bin(pub_key,
2597 				op->pub_key.data);
2598 	}
2599 
2600 	if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2601 		const BIGNUM *priv_key = NULL;
2602 
2603 		OPENSSL_LOG(DEBUG, "%s:%d updated priv key",
2604 				__func__, __LINE__);
2605 
2606 		/* get the generated keys */
2607 		get_dh_priv_key(dh_key, &priv_key);
2608 
2609 		/* provide generated private key back to user */
2610 		op->priv_key.length = BN_bn2bin(priv_key,
2611 				op->priv_key.data);
2612 	}
2613 
2614 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2615 
2616 	return 0;
2617 }
2618 #endif
2619 
2620 /* process modinv operation */
2621 static int
2622 process_openssl_modinv_op(struct rte_crypto_op *cop,
2623 		struct openssl_asym_session *sess)
2624 {
2625 	struct rte_crypto_asym_op *op = cop->asym;
2626 	BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
2627 	BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
2628 
2629 	if (unlikely(base == NULL || res == NULL)) {
2630 		BN_free(base);
2631 		BN_free(res);
2632 		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2633 		return -1;
2634 	}
2635 
2636 	base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
2637 			op->modinv.base.length, base);
2638 
2639 	if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
2640 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2641 		op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data);
2642 	} else {
2643 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2644 	}
2645 
2646 	BN_clear(res);
2647 	BN_clear(base);
2648 
2649 	return 0;
2650 }
2651 
2652 /* process modexp operation */
2653 static int
2654 process_openssl_modexp_op(struct rte_crypto_op *cop,
2655 		struct openssl_asym_session *sess)
2656 {
2657 	struct rte_crypto_asym_op *op = cop->asym;
2658 	BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
2659 	BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
2660 
2661 	if (unlikely(base == NULL || res == NULL)) {
2662 		BN_free(base);
2663 		BN_free(res);
2664 		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2665 		return -1;
2666 	}
2667 
2668 	base = BN_bin2bn((const unsigned char *)op->modex.base.data,
2669 			op->modex.base.length, base);
2670 
2671 	if (BN_mod_exp(res, base, sess->u.e.exp,
2672 				sess->u.e.mod, sess->u.e.ctx)) {
2673 		op->modex.result.length = BN_bn2bin(res, op->modex.result.data);
2674 		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2675 	} else {
2676 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2677 	}
2678 
2679 	BN_clear(res);
2680 	BN_clear(base);
2681 
2682 	return 0;
2683 }
2684 
2685 /* process rsa operations */
2686 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2687 static int
2688 process_openssl_rsa_op_evp(struct rte_crypto_op *cop,
2689 		struct openssl_asym_session *sess)
2690 {
2691 	struct rte_crypto_asym_op *op = cop->asym;
2692 	uint32_t pad = sess->u.r.pad;
2693 	uint8_t *tmp;
2694 	size_t outlen = 0;
2695 	int ret = -1;
2696 
2697 	cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2698 	EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx;
2699 	if (!rsa_ctx)
2700 		return ret;
2701 
2702 	switch (pad) {
2703 	case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2704 		pad = RSA_PKCS1_PADDING;
2705 		break;
2706 	case RTE_CRYPTO_RSA_PADDING_NONE:
2707 		pad = RSA_NO_PADDING;
2708 		break;
2709 	default:
2710 		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2711 		OPENSSL_LOG(ERR,
2712 				"rsa pad type not supported %d", pad);
2713 		return ret;
2714 	}
2715 
2716 	switch (op->rsa.op_type) {
2717 	case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2718 		if (EVP_PKEY_encrypt_init(rsa_ctx) != 1)
2719 			goto err_rsa;
2720 
2721 		if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2722 			goto err_rsa;
2723 
2724 		if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen,
2725 				op->rsa.message.data,
2726 				op->rsa.message.length) <= 0)
2727 			goto err_rsa;
2728 
2729 		if (outlen <= 0)
2730 			goto err_rsa;
2731 
2732 		if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen,
2733 				op->rsa.message.data,
2734 				op->rsa.message.length) <= 0)
2735 			goto err_rsa;
2736 		op->rsa.cipher.length = outlen;
2737 
2738 		OPENSSL_LOG(DEBUG,
2739 				"length of encrypted text %zu", outlen);
2740 		break;
2741 
2742 	case RTE_CRYPTO_ASYM_OP_DECRYPT:
2743 		if (EVP_PKEY_decrypt_init(rsa_ctx) != 1)
2744 			goto err_rsa;
2745 
2746 		if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2747 			goto err_rsa;
2748 
2749 		if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen,
2750 				op->rsa.cipher.data,
2751 				op->rsa.cipher.length) <= 0)
2752 			goto err_rsa;
2753 
2754 		if (outlen <= 0)
2755 			goto err_rsa;
2756 
2757 		if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen,
2758 				op->rsa.cipher.data,
2759 				op->rsa.cipher.length) <= 0)
2760 			goto err_rsa;
2761 		op->rsa.message.length = outlen;
2762 
2763 		OPENSSL_LOG(DEBUG, "length of decrypted text %zu", outlen);
2764 		break;
2765 
2766 	case RTE_CRYPTO_ASYM_OP_SIGN:
2767 		if (EVP_PKEY_sign_init(rsa_ctx) <= 0)
2768 			goto err_rsa;
2769 
2770 		if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2771 			goto err_rsa;
2772 
2773 		if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen,
2774 				op->rsa.message.data,
2775 				op->rsa.message.length) <= 0)
2776 			goto err_rsa;
2777 
2778 		if (outlen <= 0)
2779 			goto err_rsa;
2780 
2781 		if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen,
2782 				op->rsa.message.data,
2783 				op->rsa.message.length) <= 0)
2784 			goto err_rsa;
2785 		op->rsa.sign.length = outlen;
2786 		break;
2787 
2788 	case RTE_CRYPTO_ASYM_OP_VERIFY:
2789 		if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0)
2790 			goto err_rsa;
2791 
2792 		if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2793 			goto err_rsa;
2794 
2795 		if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen,
2796 				op->rsa.sign.data,
2797 				op->rsa.sign.length) <= 0)
2798 			goto err_rsa;
2799 
2800 		if ((outlen <= 0) || (outlen != op->rsa.sign.length))
2801 			goto err_rsa;
2802 
2803 		tmp = OPENSSL_malloc(outlen);
2804 		if (tmp == NULL) {
2805 			OPENSSL_LOG(ERR, "Memory allocation failed");
2806 			goto err_rsa;
2807 		}
2808 
2809 		if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen,
2810 				op->rsa.sign.data,
2811 				op->rsa.sign.length) <= 0) {
2812 			OPENSSL_free(tmp);
2813 			goto err_rsa;
2814 		}
2815 
2816 		OPENSSL_LOG(DEBUG,
2817 				"Length of public_decrypt %zu "
2818 				"length of message %zd",
2819 				outlen, op->rsa.message.length);
2820 		if (CRYPTO_memcmp(tmp, op->rsa.message.data,
2821 				op->rsa.message.length)) {
2822 			OPENSSL_LOG(ERR, "RSA sign Verification failed");
2823 		}
2824 		OPENSSL_free(tmp);
2825 		break;
2826 
2827 	default:
2828 		/* allow ops with invalid args to be pushed to
2829 		 * completion queue
2830 		 */
2831 		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2832 		goto err_rsa;
2833 	}
2834 
2835 	ret = 0;
2836 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2837 err_rsa:
2838 	return ret;
2839 
2840 }
2841 
2842 static int
2843 process_openssl_ecfpm_op_evp(struct rte_crypto_op *cop,
2844 		struct openssl_asym_session *sess)
2845 {
2846 	const EC_GROUP *ecgrp = sess->u.ec.group;
2847 	EC_POINT *ecpt = NULL;
2848 	BN_CTX *ctx = NULL;
2849 	BIGNUM *n = NULL;
2850 	int ret = -1;
2851 
2852 	n = BN_bin2bn((const unsigned char *)
2853 			cop->asym->ecpm.scalar.data,
2854 			cop->asym->ecpm.scalar.length,
2855 			BN_new());
2856 
2857 	ctx = BN_CTX_new();
2858 	if (!ctx)
2859 		goto err_ecfpm;
2860 
2861 	if (!EC_POINT_mul(ecgrp, ecpt, n, NULL, NULL, ctx))
2862 		goto err_ecfpm;
2863 
2864 	if (cop->asym->flags & RTE_CRYPTO_ASYM_FLAG_PUB_KEY_COMPRESSED) {
2865 		unsigned char *buf = cop->asym->ecpm.r.x.data;
2866 		size_t sz;
2867 
2868 		sz = EC_POINT_point2oct(ecgrp, ecpt, POINT_CONVERSION_COMPRESSED, buf, 0, ctx);
2869 		if (!sz)
2870 			goto err_ecfpm;
2871 
2872 		cop->asym->ecpm.r.x.length = sz;
2873 	}
2874 
2875 err_ecfpm:
2876 	BN_CTX_free(ctx);
2877 	BN_free(n);
2878 	return ret;
2879 }
2880 
2881 static int
2882 process_openssl_sm2_op_evp(struct rte_crypto_op *cop,
2883 		struct openssl_asym_session *sess)
2884 {
2885 	EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL;
2886 	struct rte_crypto_asym_op *op = cop->asym;
2887 	OSSL_PARAM *params = sess->u.sm2.params;
2888 	EVP_MD_CTX *md_ctx = NULL;
2889 	ECDSA_SIG *ec_sign = NULL;
2890 	EVP_MD *check_md = NULL;
2891 	EVP_PKEY *pkey = NULL;
2892 	int ret = -1;
2893 
2894 	cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2895 
2896 	if (cop->asym->sm2.k.data != NULL)
2897 		goto err_sm2;
2898 
2899 	switch (op->sm2.op_type) {
2900 	case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2901 		{
2902 			OSSL_PARAM *eparams = sess->u.sm2.params;
2903 			size_t output_len = 0;
2904 
2905 			kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL);
2906 			if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 ||
2907 				EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
2908 				goto err_sm2;
2909 
2910 			cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
2911 			if (!cctx)
2912 				goto err_sm2;
2913 
2914 			if (!EVP_PKEY_encrypt_init(cctx))
2915 				goto err_sm2;
2916 
2917 			if (!EVP_PKEY_CTX_set_params(cctx, eparams))
2918 				goto err_sm2;
2919 
2920 			if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len,
2921 								 op->sm2.message.data,
2922 								 op->sm2.message.length))
2923 				goto err_sm2;
2924 			op->sm2.cipher.length = output_len;
2925 		}
2926 		break;
2927 	case RTE_CRYPTO_ASYM_OP_DECRYPT:
2928 		{
2929 			OSSL_PARAM *eparams = sess->u.sm2.params;
2930 
2931 			kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL);
2932 			if (kctx == NULL
2933 				|| EVP_PKEY_fromdata_init(kctx) <= 0
2934 				|| EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
2935 				goto err_sm2;
2936 
2937 			cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
2938 			if (!cctx)
2939 				goto err_sm2;
2940 
2941 			if (!EVP_PKEY_decrypt_init(cctx))
2942 				goto err_sm2;
2943 
2944 			if (!EVP_PKEY_CTX_set_params(cctx, eparams))
2945 				goto err_sm2;
2946 
2947 			if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length,
2948 					op->sm2.cipher.data, op->sm2.cipher.length))
2949 				goto err_sm2;
2950 		}
2951 		break;
2952 	case RTE_CRYPTO_ASYM_OP_SIGN:
2953 		{
2954 			unsigned char signbuf[128] = {0};
2955 			const unsigned char *signptr;
2956 			const BIGNUM *r, *s;
2957 			size_t signlen;
2958 
2959 			kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL);
2960 			if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 ||
2961 				EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
2962 				goto err_sm2;
2963 
2964 			md_ctx = EVP_MD_CTX_new();
2965 			if (!md_ctx)
2966 				goto err_sm2;
2967 
2968 			sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
2969 			if (!sctx)
2970 				goto err_sm2;
2971 
2972 			EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx);
2973 
2974 			check_md = EVP_MD_fetch(NULL, "sm3", NULL);
2975 			if (!check_md)
2976 				goto err_sm2;
2977 
2978 			if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey))
2979 				goto err_sm2;
2980 
2981 			if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0)
2982 				goto err_sm2;
2983 
2984 			if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data,
2985 					op->sm2.message.length))
2986 				goto err_sm2;
2987 
2988 			if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen))
2989 				goto err_sm2;
2990 
2991 			if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen))
2992 				goto err_sm2;
2993 
2994 			signptr = signbuf;
2995 			ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen);
2996 			if (!ec_sign)
2997 				goto err_sm2;
2998 
2999 			r = ECDSA_SIG_get0_r(ec_sign);
3000 			s = ECDSA_SIG_get0_s(ec_sign);
3001 			if (!r || !s)
3002 				goto err_sm2;
3003 
3004 			op->sm2.r.length = BN_num_bytes(r);
3005 			op->sm2.s.length = BN_num_bytes(s);
3006 			BN_bn2bin(r, op->sm2.r.data);
3007 			BN_bn2bin(s, op->sm2.s.data);
3008 
3009 			ECDSA_SIG_free(ec_sign);
3010 		}
3011 		break;
3012 	case RTE_CRYPTO_ASYM_OP_VERIFY:
3013 		{
3014 			unsigned char signbuf[128] = {0}, *signbuf_new = NULL;
3015 			BIGNUM *r = NULL, *s = NULL;
3016 			size_t signlen;
3017 
3018 			kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL);
3019 			if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 ||
3020 				EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0)
3021 				goto err_sm2;
3022 
3023 			if (!EVP_PKEY_is_a(pkey, "SM2"))
3024 				goto err_sm2;
3025 
3026 			md_ctx = EVP_MD_CTX_new();
3027 			if (!md_ctx)
3028 				goto err_sm2;
3029 
3030 			sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
3031 			if (!sctx)
3032 				goto err_sm2;
3033 
3034 			EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx);
3035 
3036 			check_md = EVP_MD_fetch(NULL, "sm3", NULL);
3037 			if (!check_md)
3038 				goto err_sm2;
3039 
3040 			if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey))
3041 				goto err_sm2;
3042 
3043 			if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0)
3044 				goto err_sm2;
3045 
3046 			if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data,
3047 					op->sm2.message.length))
3048 				goto err_sm2;
3049 
3050 			ec_sign = ECDSA_SIG_new();
3051 			if (!ec_sign)
3052 				goto err_sm2;
3053 
3054 			r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r);
3055 			s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s);
3056 			if (!r || !s)
3057 				goto err_sm2;
3058 
3059 			if (!ECDSA_SIG_set0(ec_sign, r, s)) {
3060 				BN_free(r);
3061 				BN_free(s);
3062 				goto err_sm2;
3063 			}
3064 
3065 			r = NULL;
3066 			s = NULL;
3067 
3068 			signbuf_new = signbuf;
3069 			signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf_new);
3070 			if (signlen <= 0)
3071 				goto err_sm2;
3072 
3073 			if (!EVP_DigestVerifyFinal(md_ctx, signbuf_new, signlen))
3074 				goto err_sm2;
3075 
3076 			BN_free(r);
3077 			BN_free(s);
3078 			ECDSA_SIG_free(ec_sign);
3079 	}
3080 		break;
3081 	default:
3082 		/* allow ops with invalid args to be pushed to
3083 		 * completion queue
3084 		 */
3085 		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
3086 		goto err_sm2;
3087 	}
3088 
3089 	ret = 0;
3090 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3091 err_sm2:
3092 	EVP_MD_free(check_md);
3093 	EVP_MD_CTX_free(md_ctx);
3094 
3095 	EVP_PKEY_CTX_free(kctx);
3096 
3097 	EVP_PKEY_CTX_free(sctx);
3098 
3099 	EVP_PKEY_CTX_free(cctx);
3100 
3101 	EVP_PKEY_free(pkey);
3102 
3103 	return ret;
3104 }
3105 
3106 static int
3107 process_openssl_eddsa_op_evp(struct rte_crypto_op *cop,
3108 		struct openssl_asym_session *sess)
3109 {
3110 	static const char * const instance[] = {"Ed25519", "Ed25519ctx", "Ed25519ph",
3111 						"Ed448", "Ed448ph"};
3112 	EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL;
3113 	const uint8_t curve_id = sess->u.eddsa.curve_id;
3114 	struct rte_crypto_asym_op *op = cop->asym;
3115 	OSSL_PARAM *params = sess->u.eddsa.params;
3116 	OSSL_PARAM_BLD *iparam_bld = NULL;
3117 	OSSL_PARAM *iparams = NULL;
3118 	uint8_t signbuf[128] = {0};
3119 	EVP_MD_CTX *md_ctx = NULL;
3120 	EVP_PKEY *pkey = NULL;
3121 	size_t signlen;
3122 	int ret = -1;
3123 
3124 	cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
3125 
3126 	iparam_bld = OSSL_PARAM_BLD_new();
3127 	if (!iparam_bld)
3128 		goto err_eddsa;
3129 
3130 	if (op->eddsa.instance == RTE_CRYPTO_EDCURVE_25519CTX) {
3131 		OSSL_PARAM_BLD_push_octet_string(iparam_bld, "context-string",
3132 			op->eddsa.context.data, op->eddsa.context.length);
3133 
3134 	}
3135 
3136 	OSSL_PARAM_BLD_push_utf8_string(iparam_bld, "instance",
3137 		instance[op->eddsa.instance], strlen(instance[op->eddsa.instance]));
3138 
3139 	iparams = OSSL_PARAM_BLD_to_param(iparam_bld);
3140 	if (!iparams)
3141 		goto err_eddsa;
3142 
3143 	switch (op->eddsa.op_type) {
3144 	case RTE_CRYPTO_ASYM_OP_SIGN:
3145 		{
3146 			if (curve_id == RTE_CRYPTO_EC_GROUP_ED25519)
3147 				kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED25519", NULL);
3148 			else
3149 				kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED448", NULL);
3150 
3151 			if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 ||
3152 				EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
3153 				goto err_eddsa;
3154 
3155 			md_ctx = EVP_MD_CTX_new();
3156 			if (!md_ctx)
3157 				goto err_eddsa;
3158 
3159 			sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
3160 			if (!sctx)
3161 				goto err_eddsa;
3162 
3163 			EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx);
3164 
3165 #if (OPENSSL_VERSION_NUMBER >= 0x30300000L)
3166 			if (!EVP_DigestSignInit_ex(md_ctx, NULL, NULL, NULL, NULL, pkey, iparams))
3167 				goto err_eddsa;
3168 #else
3169 			if (op->eddsa.instance == RTE_CRYPTO_EDCURVE_25519 ||
3170 				op->eddsa.instance == RTE_CRYPTO_EDCURVE_448) {
3171 				if (!EVP_DigestSignInit(md_ctx, NULL, NULL, NULL, pkey))
3172 					goto err_eddsa;
3173 			} else
3174 				goto err_eddsa;
3175 #endif
3176 
3177 			if (!EVP_DigestSign(md_ctx, NULL, &signlen, op->eddsa.message.data,
3178 					op->eddsa.message.length))
3179 				goto err_eddsa;
3180 
3181 			if (signlen > RTE_DIM(signbuf))
3182 				goto err_eddsa;
3183 
3184 			if (!EVP_DigestSign(md_ctx, signbuf, &signlen, op->eddsa.message.data,
3185 					op->eddsa.message.length))
3186 				goto err_eddsa;
3187 
3188 			memcpy(op->eddsa.sign.data, &signbuf[0], signlen);
3189 			op->eddsa.sign.length = signlen;
3190 		}
3191 		break;
3192 	case RTE_CRYPTO_ASYM_OP_VERIFY:
3193 		{
3194 			if (curve_id == RTE_CRYPTO_EC_GROUP_ED25519)
3195 				kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED25519", NULL);
3196 			else
3197 				kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED448", NULL);
3198 
3199 			if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 ||
3200 				EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0)
3201 				goto err_eddsa;
3202 
3203 			md_ctx = EVP_MD_CTX_new();
3204 			if (!md_ctx)
3205 				goto err_eddsa;
3206 
3207 			sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
3208 			if (!sctx)
3209 				goto err_eddsa;
3210 
3211 			EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx);
3212 
3213 #if (OPENSSL_VERSION_NUMBER >= 0x30300000L)
3214 			if (!EVP_DigestVerifyInit_ex(md_ctx, NULL, NULL, NULL, NULL, pkey, iparams))
3215 				goto err_eddsa;
3216 #else
3217 			if (op->eddsa.instance == RTE_CRYPTO_EDCURVE_25519 ||
3218 				op->eddsa.instance == RTE_CRYPTO_EDCURVE_448) {
3219 				if (!EVP_DigestVerifyInit(md_ctx, NULL, NULL, NULL, pkey))
3220 					goto err_eddsa;
3221 			} else
3222 				goto err_eddsa;
3223 #endif
3224 
3225 			signlen = op->eddsa.sign.length;
3226 			memcpy(&signbuf[0], op->eddsa.sign.data, op->eddsa.sign.length);
3227 
3228 			ret = EVP_DigestVerify(md_ctx, signbuf, signlen, op->eddsa.message.data,
3229 					op->eddsa.message.length);
3230 			if (ret == 0)
3231 				goto err_eddsa;
3232 		}
3233 		break;
3234 	default:
3235 		/* allow ops with invalid args to be pushed to
3236 		 * completion queue
3237 		 */
3238 		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
3239 		goto err_eddsa;
3240 	}
3241 
3242 	ret = 0;
3243 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3244 err_eddsa:
3245 	OSSL_PARAM_BLD_free(iparam_bld);
3246 
3247 	EVP_PKEY_CTX_free(sctx);
3248 
3249 	EVP_PKEY_CTX_free(cctx);
3250 
3251 	EVP_PKEY_free(pkey);
3252 
3253 	return ret;
3254 }
3255 #else
3256 static int
3257 process_openssl_rsa_op(struct rte_crypto_op *cop,
3258 		struct openssl_asym_session *sess)
3259 {
3260 	int ret = 0;
3261 	struct rte_crypto_asym_op *op = cop->asym;
3262 	RSA *rsa = sess->u.r.rsa;
3263 	uint32_t pad = sess->u.r.pad;
3264 	uint8_t *tmp;
3265 
3266 	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3267 
3268 	switch (pad) {
3269 	case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
3270 		pad = RSA_PKCS1_PADDING;
3271 		break;
3272 	case RTE_CRYPTO_RSA_PADDING_NONE:
3273 		pad = RSA_NO_PADDING;
3274 		break;
3275 	default:
3276 		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
3277 		OPENSSL_LOG(ERR,
3278 				"rsa pad type not supported %d", pad);
3279 		return 0;
3280 	}
3281 
3282 	switch (op->rsa.op_type) {
3283 	case RTE_CRYPTO_ASYM_OP_ENCRYPT:
3284 		ret = RSA_public_encrypt(op->rsa.message.length,
3285 				op->rsa.message.data,
3286 				op->rsa.cipher.data,
3287 				rsa,
3288 				pad);
3289 
3290 		if (ret > 0)
3291 			op->rsa.cipher.length = ret;
3292 		OPENSSL_LOG(DEBUG,
3293 				"length of encrypted text %d", ret);
3294 		break;
3295 
3296 	case RTE_CRYPTO_ASYM_OP_DECRYPT:
3297 		ret = RSA_private_decrypt(op->rsa.cipher.length,
3298 				op->rsa.cipher.data,
3299 				op->rsa.message.data,
3300 				rsa,
3301 				pad);
3302 		if (ret > 0)
3303 			op->rsa.message.length = ret;
3304 		break;
3305 
3306 	case RTE_CRYPTO_ASYM_OP_SIGN:
3307 		ret = RSA_private_encrypt(op->rsa.message.length,
3308 				op->rsa.message.data,
3309 				op->rsa.sign.data,
3310 				rsa,
3311 				pad);
3312 		if (ret > 0)
3313 			op->rsa.sign.length = ret;
3314 		break;
3315 
3316 	case RTE_CRYPTO_ASYM_OP_VERIFY:
3317 		tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
3318 		if (tmp == NULL) {
3319 			OPENSSL_LOG(ERR, "Memory allocation failed");
3320 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
3321 			break;
3322 		}
3323 		ret = RSA_public_decrypt(op->rsa.sign.length,
3324 				op->rsa.sign.data,
3325 				tmp,
3326 				rsa,
3327 				pad);
3328 
3329 		OPENSSL_LOG(DEBUG,
3330 				"Length of public_decrypt %d "
3331 				"length of message %zd",
3332 				ret, op->rsa.message.length);
3333 		if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data,
3334 				op->rsa.message.length))) {
3335 			OPENSSL_LOG(ERR, "RSA sign Verification failed");
3336 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
3337 		}
3338 		rte_free(tmp);
3339 		break;
3340 
3341 	default:
3342 		/* allow ops with invalid args to be pushed to
3343 		 * completion queue
3344 		 */
3345 		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
3346 		break;
3347 	}
3348 
3349 	if (ret < 0)
3350 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
3351 
3352 	return 0;
3353 }
3354 
3355 static int
3356 process_openssl_ecfpm_op(struct rte_crypto_op *cop,
3357 		struct openssl_asym_session *sess)
3358 {
3359 	RTE_SET_USED(cop);
3360 	RTE_SET_USED(sess);
3361 	return -ENOTSUP;
3362 }
3363 
3364 static int
3365 process_openssl_sm2_op(struct rte_crypto_op *cop,
3366 		struct openssl_asym_session *sess)
3367 {
3368 	RTE_SET_USED(cop);
3369 	RTE_SET_USED(sess);
3370 	return -ENOTSUP;
3371 }
3372 
3373 static int
3374 process_openssl_eddsa_op(struct rte_crypto_op *cop,
3375 		struct openssl_asym_session *sess)
3376 {
3377 	RTE_SET_USED(cop);
3378 	RTE_SET_USED(sess);
3379 	return -ENOTSUP;
3380 }
3381 #endif
3382 
3383 static int
3384 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
3385 		struct openssl_asym_session *sess)
3386 {
3387 	int retval = 0;
3388 
3389 	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
3390 
3391 	switch (sess->xfrm_type) {
3392 	case RTE_CRYPTO_ASYM_XFORM_RSA:
3393 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3394 		retval = process_openssl_rsa_op_evp(op, sess);
3395 # else
3396 		retval = process_openssl_rsa_op(op, sess);
3397 #endif
3398 		break;
3399 	case RTE_CRYPTO_ASYM_XFORM_MODEX:
3400 		retval = process_openssl_modexp_op(op, sess);
3401 		break;
3402 	case RTE_CRYPTO_ASYM_XFORM_MODINV:
3403 		retval = process_openssl_modinv_op(op, sess);
3404 		break;
3405 	case RTE_CRYPTO_ASYM_XFORM_DH:
3406 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3407 		retval = process_openssl_dh_op_evp(op, sess);
3408 # else
3409 		retval = process_openssl_dh_op(op, sess);
3410 #endif
3411 		break;
3412 	case RTE_CRYPTO_ASYM_XFORM_DSA:
3413 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3414 		if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
3415 			retval = process_openssl_dsa_sign_op_evp(op, sess);
3416 		else if (op->asym->dsa.op_type ==
3417 				RTE_CRYPTO_ASYM_OP_VERIFY)
3418 			retval =
3419 				process_openssl_dsa_verify_op_evp(op, sess);
3420 #else
3421 		if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
3422 			retval = process_openssl_dsa_sign_op(op, sess);
3423 		else if (op->asym->dsa.op_type ==
3424 				RTE_CRYPTO_ASYM_OP_VERIFY)
3425 			retval =
3426 				process_openssl_dsa_verify_op(op, sess);
3427 		else
3428 			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
3429 #endif
3430 		break;
3431 	case RTE_CRYPTO_ASYM_XFORM_ECFPM:
3432 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3433 		retval = process_openssl_ecfpm_op_evp(op, sess);
3434 #else
3435 		retval = process_openssl_ecfpm_op(op, sess);
3436 #endif
3437 		break;
3438 	case RTE_CRYPTO_ASYM_XFORM_SM2:
3439 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3440 		retval = process_openssl_sm2_op_evp(op, sess);
3441 #else
3442 		retval = process_openssl_sm2_op(op, sess);
3443 #endif
3444 		break;
3445 	case RTE_CRYPTO_ASYM_XFORM_EDDSA:
3446 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3447 		retval = process_openssl_eddsa_op_evp(op, sess);
3448 #else
3449 		retval = process_openssl_eddsa_op(op, sess);
3450 #endif
3451 		break;
3452 	default:
3453 		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
3454 		break;
3455 	}
3456 	if (!retval) {
3457 		/* op processed so push to completion queue as processed */
3458 		retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
3459 		if (retval)
3460 			/* return error if failed to put in completion queue */
3461 			retval = -1;
3462 	}
3463 
3464 	return retval;
3465 }
3466 
3467 static void
3468 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst,
3469 		struct rte_crypto_op *op)
3470 {
3471 	uint8_t *p_src, *p_dst;
3472 
3473 	p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
3474 	p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
3475 
3476 	/**
3477 	 * Copy the content between cipher offset and auth offset
3478 	 * for generating correct digest.
3479 	 */
3480 	if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
3481 		memcpy(p_dst + op->sym->auth.data.offset,
3482 				p_src + op->sym->auth.data.offset,
3483 				op->sym->cipher.data.offset -
3484 				op->sym->auth.data.offset);
3485 }
3486 
3487 /** Process crypto operation for mbuf */
3488 static int
3489 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
3490 		struct openssl_session *sess)
3491 {
3492 	struct rte_mbuf *msrc, *mdst;
3493 	int retval;
3494 
3495 	msrc = op->sym->m_src;
3496 	mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
3497 
3498 	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
3499 
3500 	switch (sess->chain_order) {
3501 	case OPENSSL_CHAIN_ONLY_CIPHER:
3502 		process_openssl_cipher_op(qp, op, sess, msrc, mdst);
3503 		break;
3504 	case OPENSSL_CHAIN_ONLY_AUTH:
3505 		process_openssl_auth_op(qp, op, sess, msrc, mdst);
3506 		break;
3507 	case OPENSSL_CHAIN_CIPHER_AUTH:
3508 		process_openssl_cipher_op(qp, op, sess, msrc, mdst);
3509 		/* OOP */
3510 		if (msrc != mdst)
3511 			copy_plaintext(msrc, mdst, op);
3512 		process_openssl_auth_op(qp, op, sess, mdst, mdst);
3513 		break;
3514 	case OPENSSL_CHAIN_AUTH_CIPHER:
3515 		process_openssl_auth_op(qp, op, sess, msrc, mdst);
3516 		process_openssl_cipher_op(qp, op, sess, msrc, mdst);
3517 		break;
3518 	case OPENSSL_CHAIN_COMBINED:
3519 		process_openssl_combined_op(qp, op, sess, msrc, mdst);
3520 		break;
3521 	case OPENSSL_CHAIN_CIPHER_BPI:
3522 		process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
3523 		break;
3524 	default:
3525 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3526 		break;
3527 	}
3528 
3529 	/* Free session if a session-less crypto op */
3530 	if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
3531 		openssl_reset_session(sess);
3532 		memset(sess, 0, sizeof(struct openssl_session));
3533 		rte_mempool_put(qp->sess_mp, op->sym->session);
3534 		op->sym->session = NULL;
3535 	}
3536 
3537 	if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
3538 		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3539 
3540 	if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
3541 		retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
3542 	else
3543 		retval = -1;
3544 
3545 	return retval;
3546 }
3547 
3548 /*
3549  *------------------------------------------------------------------------------
3550  * PMD Framework
3551  *------------------------------------------------------------------------------
3552  */
3553 
3554 /** Enqueue burst */
3555 static uint16_t
3556 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
3557 		uint16_t nb_ops)
3558 {
3559 	void *sess;
3560 	struct openssl_qp *qp = queue_pair;
3561 	int i, retval;
3562 
3563 	for (i = 0; i < nb_ops; i++) {
3564 		sess = get_session(qp, ops[i]);
3565 		if (unlikely(sess == NULL))
3566 			goto enqueue_err;
3567 
3568 		if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
3569 			retval = process_op(qp, ops[i],
3570 					(struct openssl_session *) sess);
3571 		else
3572 			retval = process_asym_op(qp, ops[i],
3573 					(struct openssl_asym_session *) sess);
3574 		if (unlikely(retval < 0))
3575 			goto enqueue_err;
3576 	}
3577 
3578 	qp->stats.enqueued_count += i;
3579 	return i;
3580 
3581 enqueue_err:
3582 	qp->stats.enqueue_err_count++;
3583 	return i;
3584 }
3585 
3586 /** Dequeue burst */
3587 static uint16_t
3588 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
3589 		uint16_t nb_ops)
3590 {
3591 	struct openssl_qp *qp = queue_pair;
3592 
3593 	unsigned int nb_dequeued = 0;
3594 
3595 	nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
3596 			(void **)ops, nb_ops, NULL);
3597 	qp->stats.dequeued_count += nb_dequeued;
3598 
3599 	return nb_dequeued;
3600 }
3601 
3602 /** Create OPENSSL crypto device */
3603 static int
3604 cryptodev_openssl_create(const char *name,
3605 			struct rte_vdev_device *vdev,
3606 			struct rte_cryptodev_pmd_init_params *init_params)
3607 {
3608 	struct rte_cryptodev *dev;
3609 	struct openssl_private *internals;
3610 
3611 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
3612 	if (dev == NULL) {
3613 		OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
3614 		goto init_error;
3615 	}
3616 
3617 	dev->driver_id = cryptodev_driver_id;
3618 	dev->dev_ops = rte_openssl_pmd_ops;
3619 
3620 	/* register rx/tx burst functions for data path */
3621 	dev->dequeue_burst = openssl_pmd_dequeue_burst;
3622 	dev->enqueue_burst = openssl_pmd_enqueue_burst;
3623 
3624 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
3625 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
3626 			RTE_CRYPTODEV_FF_CPU_AESNI |
3627 			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
3628 			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
3629 			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
3630 			RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
3631 			RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
3632 			RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT |
3633 			RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
3634 
3635 	internals = dev->data->dev_private;
3636 
3637 	internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
3638 
3639 	rte_cryptodev_pmd_probing_finish(dev);
3640 
3641 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3642 	/* Load legacy provider
3643 	 * Some algorithms are no longer available in earlier version of openssl,
3644 	 * unless the legacy provider explicitly loaded. e.g. DES
3645 	 */
3646 	ossl_legacy_provider_load();
3647 # endif
3648 	return 0;
3649 
3650 init_error:
3651 	OPENSSL_LOG(ERR, "driver %s: create failed",
3652 			init_params->name);
3653 
3654 	cryptodev_openssl_remove(vdev);
3655 	return -EFAULT;
3656 }
3657 
3658 /** Initialise OPENSSL crypto device */
3659 static int
3660 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
3661 {
3662 	struct rte_cryptodev_pmd_init_params init_params = {
3663 		"",
3664 		sizeof(struct openssl_private),
3665 		rte_socket_id(),
3666 		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
3667 	};
3668 	const char *name;
3669 	const char *input_args;
3670 
3671 	name = rte_vdev_device_name(vdev);
3672 	if (name == NULL)
3673 		return -EINVAL;
3674 	input_args = rte_vdev_device_args(vdev);
3675 
3676 	rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
3677 
3678 	return cryptodev_openssl_create(name, vdev, &init_params);
3679 }
3680 
3681 /** Uninitialise OPENSSL crypto device */
3682 static int
3683 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
3684 {
3685 	struct rte_cryptodev *cryptodev;
3686 	const char *name;
3687 
3688 	name = rte_vdev_device_name(vdev);
3689 	if (name == NULL)
3690 		return -EINVAL;
3691 
3692 	cryptodev = rte_cryptodev_pmd_get_named_dev(name);
3693 	if (cryptodev == NULL)
3694 		return -ENODEV;
3695 
3696 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
3697 	ossl_legacy_provider_unload();
3698 # endif
3699 	return rte_cryptodev_pmd_destroy(cryptodev);
3700 }
3701 
3702 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
3703 	.probe = cryptodev_openssl_probe,
3704 	.remove = cryptodev_openssl_remove
3705 };
3706 
3707 static struct cryptodev_driver openssl_crypto_drv;
3708 
3709 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
3710 	cryptodev_openssl_pmd_drv);
3711 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
3712 	"max_nb_queue_pairs=<int> "
3713 	"socket_id=<int>");
3714 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
3715 		cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
3716 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO);
3717