xref: /onnv-gate/usr/src/common/openssl/crypto/engine/hw_pk11_pub.c (revision 11411:c2fe1bf96826)
1 /*
2  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 
6 /* crypto/engine/hw_pk11_pub.c */
7 /*
8  * This product includes software developed by the OpenSSL Project for
9  * use in the OpenSSL Toolkit (http://www.openssl.org/).
10  *
11  * This project also referenced hw_pkcs11-0.9.7b.patch written by
12  * Afchine Madjlessi.
13  */
14 /*
15  * ====================================================================
16  * Copyright (c) 2000-2001 The OpenSSL Project.  All rights reserved.
17  *
18  * Redistribution and use in source and binary forms, with or without
19  * modification, are permitted provided that the following conditions
20  * are met:
21  *
22  * 1. Redistributions of source code must retain the above copyright
23  *    notice, this list of conditions and the following disclaimer.
24  *
25  * 2. Redistributions in binary form must reproduce the above copyright
26  *    notice, this list of conditions and the following disclaimer in
27  *    the documentation and/or other materials provided with the
28  *    distribution.
29  *
30  * 3. All advertising materials mentioning features or use of this
31  *    software must display the following acknowledgment:
32  *    "This product includes software developed by the OpenSSL Project
33  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
34  *
35  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
36  *    endorse or promote products derived from this software without
37  *    prior written permission. For written permission, please contact
38  *    licensing@OpenSSL.org.
39  *
40  * 5. Products derived from this software may not be called "OpenSSL"
41  *    nor may "OpenSSL" appear in their names without prior written
42  *    permission of the OpenSSL Project.
43  *
44  * 6. Redistributions of any form whatsoever must retain the following
45  *    acknowledgment:
46  *    "This product includes software developed by the OpenSSL Project
47  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
48  *
49  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
50  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
52  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
53  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
55  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
56  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
58  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
60  * OF THE POSSIBILITY OF SUCH DAMAGE.
61  * ====================================================================
62  *
63  * This product includes cryptographic software written by Eric Young
64  * (eay@cryptsoft.com).  This product includes software written by Tim
65  * Hudson (tjh@cryptsoft.com).
66  *
67  */
68 
69 #include <stdio.h>
70 #include <stdlib.h>
71 #include <string.h>
72 #include <sys/types.h>
73 #include <unistd.h>
74 
75 #include <openssl/e_os2.h>
76 #include <openssl/crypto.h>
77 #include <openssl/engine.h>
78 #include <openssl/dso.h>
79 #include <openssl/err.h>
80 #include <openssl/bn.h>
81 #include <openssl/pem.h>
82 #ifndef OPENSSL_NO_RSA
83 #include <openssl/rsa.h>
84 #endif /* OPENSSL_NO_RSA */
85 #ifndef OPENSSL_NO_DSA
86 #include <openssl/dsa.h>
87 #endif /* OPENSSL_NO_DSA */
88 #ifndef OPENSSL_NO_DH
89 #include <openssl/dh.h>
90 #endif /* OPENSSL_NO_DH */
91 #include <openssl/rand.h>
92 #include <openssl/objects.h>
93 #include <openssl/x509.h>
94 #include <cryptlib.h>
95 #include <pthread.h>
96 
97 #ifndef OPENSSL_NO_HW
98 #ifndef OPENSSL_NO_HW_PK11
99 
100 #include "security/cryptoki.h"
101 #include "security/pkcs11.h"
102 #include "hw_pk11_err.h"
103 
104 #ifndef OPENSSL_NO_RSA
105 /* RSA stuff */
106 static int pk11_RSA_public_encrypt(int flen, const unsigned char *from,
107 	unsigned char *to, RSA *rsa, int padding);
108 static int pk11_RSA_private_encrypt(int flen, const unsigned char *from,
109 	unsigned char *to, RSA *rsa, int padding);
110 static int pk11_RSA_public_decrypt(int flen, const unsigned char *from,
111 	unsigned char *to, RSA *rsa, int padding);
112 static int pk11_RSA_private_decrypt(int flen, const unsigned char *from,
113 	unsigned char *to, RSA *rsa, int padding);
114 static int pk11_RSA_init(RSA *rsa);
115 static int pk11_RSA_finish(RSA *rsa);
116 static int pk11_RSA_sign(int type, const unsigned char *m, unsigned int m_len,
117 	unsigned char *sigret, unsigned int *siglen, const RSA *rsa);
118 static int pk11_RSA_verify(int dtype, const unsigned char *m,
119 	unsigned int m_len, unsigned char *sigbuf, unsigned int siglen,
120 	const RSA *rsa);
121 EVP_PKEY *pk11_load_privkey(ENGINE*, const char *pubkey_file,
122 	UI_METHOD *ui_method, void *callback_data);
123 EVP_PKEY *pk11_load_pubkey(ENGINE*, const char *pubkey_file,
124 	UI_METHOD *ui_method, void *callback_data);
125 
126 static int pk11_RSA_public_encrypt_low(int flen, const unsigned char *from,
127 	unsigned char *to, RSA *rsa);
128 static int pk11_RSA_private_encrypt_low(int flen, const unsigned char *from,
129 	unsigned char *to, RSA *rsa);
130 static int pk11_RSA_public_decrypt_low(int flen, const unsigned char *from,
131 	unsigned char *to, RSA *rsa);
132 static int pk11_RSA_private_decrypt_low(int flen, const unsigned char *from,
133 	unsigned char *to, RSA *rsa);
134 
135 static CK_OBJECT_HANDLE pk11_get_public_rsa_key(RSA* rsa, RSA** key_ptr,
136 	BIGNUM **rsa_n_num, BIGNUM **rsa_e_num, CK_SESSION_HANDLE session);
137 static CK_OBJECT_HANDLE pk11_get_private_rsa_key(RSA* rsa, RSA** key_ptr,
138 	BIGNUM **rsa_d_num, CK_SESSION_HANDLE session);
139 
140 static int check_new_rsa_key_pub(PK11_SESSION *sp, const RSA *rsa);
141 static int check_new_rsa_key_priv(PK11_SESSION *sp, const RSA *rsa);
142 #endif
143 
144 /* DSA stuff */
145 #ifndef OPENSSL_NO_DSA
146 static int pk11_DSA_init(DSA *dsa);
147 static int pk11_DSA_finish(DSA *dsa);
148 static DSA_SIG *pk11_dsa_do_sign(const unsigned char *dgst, int dlen,
149 	DSA *dsa);
150 static int pk11_dsa_do_verify(const unsigned char *dgst, int dgst_len,
151 	DSA_SIG *sig, DSA *dsa);
152 
153 static CK_OBJECT_HANDLE pk11_get_public_dsa_key(DSA* dsa, DSA **key_ptr,
154 	BIGNUM **dsa_pub_num, CK_SESSION_HANDLE session);
155 static CK_OBJECT_HANDLE pk11_get_private_dsa_key(DSA* dsa, DSA **key_ptr,
156 	BIGNUM **dsa_priv_num, CK_SESSION_HANDLE session);
157 
158 static int check_new_dsa_key_pub(PK11_SESSION *sp, DSA *dsa);
159 static int check_new_dsa_key_priv(PK11_SESSION *sp, DSA *dsa);
160 #endif
161 
162 /* DH stuff */
163 #ifndef OPENSSL_NO_DH
164 static int pk11_DH_init(DH *dh);
165 static int pk11_DH_finish(DH *dh);
166 static int pk11_DH_generate_key(DH *dh);
167 static int pk11_DH_compute_key(unsigned char *key,
168 	const BIGNUM *pub_key, DH *dh);
169 
170 static CK_OBJECT_HANDLE pk11_get_dh_key(DH* dh, DH **key_ptr,
171 	BIGNUM **priv_key, CK_SESSION_HANDLE session);
172 
173 static int check_new_dh_key(PK11_SESSION *sp, DH *dh);
174 #endif
175 
176 static int init_template_value(BIGNUM *bn, CK_VOID_PTR *pValue,
177 	CK_ULONG *ulValueLen);
178 
179 /* Read mode string to be used for fopen() */
180 #if SOLARIS_OPENSSL
181 static char *read_mode_flags = "rF";
182 #else
183 static char *read_mode_flags = "r";
184 #endif
185 
186 /*
187  * increment/create reference for an asymmetric key handle via active list
188  * manipulation. If active list operation fails, unlock (if locked), set error
189  * variable and jump to the specified label.
190  */
191 #define	KEY_HANDLE_REFHOLD(key_handle, alg_type, unlock, var, label)	\
192 	{								\
193 	if (pk11_active_add(key_handle, alg_type) < 0)			\
194 		{							\
195 		var = TRUE;						\
196 		if (unlock)						\
197 			UNLOCK_OBJSTORE(alg_type);			\
198 		goto label;						\
199 		}							\
200 	}
201 
202 /*
203  * Find active list entry according to object handle and return pointer to the
204  * entry otherwise return NULL.
205  *
206  * This function presumes it is called with lock protecting the active list
207  * held.
208  */
pk11_active_find(CK_OBJECT_HANDLE h,PK11_OPTYPE type)209 static PK11_active *pk11_active_find(CK_OBJECT_HANDLE h, PK11_OPTYPE type)
210 	{
211 	PK11_active *entry;
212 
213 	for (entry = active_list[type]; entry != NULL; entry = entry->next)
214 		if (entry->h == h)
215 			return (entry);
216 
217 	return (NULL);
218 	}
219 
220 /*
221  * Search for an entry in the active list using PKCS#11 object handle as a
222  * search key and return refcnt of the found/created entry or -1 in case of
223  * failure.
224  *
225  * This function presumes it is called with lock protecting the active list
226  * held.
227  */
228 int
pk11_active_add(CK_OBJECT_HANDLE h,PK11_OPTYPE type)229 pk11_active_add(CK_OBJECT_HANDLE h, PK11_OPTYPE type)
230 	{
231 	PK11_active *entry = NULL;
232 
233 	if (h == CK_INVALID_HANDLE)
234 		{
235 		PK11err(PK11_F_ACTIVE_ADD, PK11_R_INVALID_HANDLE);
236 		return (-1);
237 		}
238 
239 	/* search for entry in the active list */
240 	if ((entry = pk11_active_find(h, type)) != NULL)
241 		entry->refcnt++;
242 	else
243 		{
244 		/* not found, create new entry and add it to the list */
245 		entry = OPENSSL_malloc(sizeof (PK11_active));
246 		if (entry == NULL)
247 			{
248 			PK11err(PK11_F_ACTIVE_ADD, PK11_R_MALLOC_FAILURE);
249 			return (-1);
250 			}
251 		entry->h = h;
252 		entry->refcnt = 1;
253 		entry->prev = NULL;
254 		entry->next = NULL;
255 		/* connect the newly created entry to the list */
256 		if (active_list[type] == NULL)
257 			active_list[type] = entry;
258 		else /* make the entry first in the list */
259 			{
260 			entry->next = active_list[type];
261 			active_list[type]->prev = entry;
262 			active_list[type] = entry;
263 			}
264 		}
265 
266 	return (entry->refcnt);
267 	}
268 
269 /*
270  * Remove active list entry from the list and free it.
271  *
272  * This function presumes it is called with lock protecting the active list
273  * held.
274  */
275 void
pk11_active_remove(PK11_active * entry,PK11_OPTYPE type)276 pk11_active_remove(PK11_active *entry, PK11_OPTYPE type)
277 	{
278 	PK11_active *prev_entry;
279 
280 	/* remove the entry from the list and free it */
281 	if ((prev_entry = entry->prev) != NULL)
282 		{
283 		prev_entry->next = entry->next;
284 		if (entry->next != NULL)
285 			entry->next->prev = prev_entry;
286 		}
287 	else
288 		{
289 		active_list[type] = entry->next;
290 		/* we were the first but not the only one */
291 		if (entry->next != NULL)
292 			entry->next->prev = NULL;
293 		}
294 
295 	/* sanitization */
296 	entry->h = CK_INVALID_HANDLE;
297 	entry->prev = NULL;
298 	entry->next = NULL;
299 	OPENSSL_free(entry);
300 	}
301 
302 /* Free all entries from the active list. */
303 void
pk11_free_active_list(PK11_OPTYPE type)304 pk11_free_active_list(PK11_OPTYPE type)
305 	{
306 	PK11_active *entry;
307 
308 	/* only for asymmetric types since only they have C_Find* locks. */
309 	switch (type)
310 		{
311 		case OP_RSA:
312 		case OP_DSA:
313 		case OP_DH:
314 			break;
315 		default:
316 			return;
317 		}
318 
319 	/* see find_lock array definition for more info on object locking */
320 	LOCK_OBJSTORE(type);
321 	while ((entry = active_list[type]) != NULL)
322 		pk11_active_remove(entry, type);
323 	UNLOCK_OBJSTORE(type);
324 	}
325 
326 /*
327  * Search for active list entry associated with given PKCS#11 object handle,
328  * decrement its refcnt and if it drops to 0, disconnect the entry and free it.
329  *
330  * Return 1 if the PKCS#11 object associated with the entry has no references,
331  * return 0 if there is at least one reference, -1 on error.
332  *
333  * This function presumes it is called with lock protecting the active list
334  * held.
335  */
336 int
pk11_active_delete(CK_OBJECT_HANDLE h,PK11_OPTYPE type)337 pk11_active_delete(CK_OBJECT_HANDLE h, PK11_OPTYPE type)
338 	{
339 	PK11_active *entry = NULL;
340 
341 	if ((entry = pk11_active_find(h, type)) == NULL)
342 		{
343 		PK11err(PK11_F_ACTIVE_DELETE, PK11_R_INVALID_HANDLE);
344 		return (-1);
345 		}
346 
347 	OPENSSL_assert(entry->refcnt > 0);
348 	entry->refcnt--;
349 	if (entry->refcnt == 0)
350 		{
351 		pk11_active_remove(entry, type);
352 		return (1);
353 		}
354 
355 	return (0);
356 	}
357 
358 #ifndef OPENSSL_NO_RSA
359 /* Our internal RSA_METHOD that we provide pointers to */
360 static RSA_METHOD pk11_rsa =
361 	{
362 	"PKCS#11 RSA method",
363 	pk11_RSA_public_encrypt,		/* rsa_pub_encrypt */
364 	pk11_RSA_public_decrypt,		/* rsa_pub_decrypt */
365 	pk11_RSA_private_encrypt,		/* rsa_priv_encrypt */
366 	pk11_RSA_private_decrypt,		/* rsa_priv_decrypt */
367 	NULL,					/* rsa_mod_exp */
368 	NULL,					/* bn_mod_exp */
369 	pk11_RSA_init,				/* init */
370 	pk11_RSA_finish,			/* finish */
371 	RSA_FLAG_SIGN_VER,			/* flags */
372 	NULL,					/* app_data */
373 	pk11_RSA_sign,				/* rsa_sign */
374 	pk11_RSA_verify				/* rsa_verify */
375 	};
376 
377 RSA_METHOD *
PK11_RSA(void)378 PK11_RSA(void)
379 	{
380 	return (&pk11_rsa);
381 	}
382 #endif
383 
384 #ifndef OPENSSL_NO_DSA
385 /* Our internal DSA_METHOD that we provide pointers to */
386 static DSA_METHOD pk11_dsa =
387 	{
388 	"PKCS#11 DSA method",
389 	pk11_dsa_do_sign, 	/* dsa_do_sign */
390 	NULL, 			/* dsa_sign_setup */
391 	pk11_dsa_do_verify, 	/* dsa_do_verify */
392 	NULL,			/* dsa_mod_exp */
393 	NULL, 			/* bn_mod_exp */
394 	pk11_DSA_init, 		/* init */
395 	pk11_DSA_finish, 	/* finish */
396 	0, 			/* flags */
397 	NULL 			/* app_data */
398 	};
399 
400 DSA_METHOD *
PK11_DSA(void)401 PK11_DSA(void)
402 	{
403 	return (&pk11_dsa);
404 	}
405 #endif
406 
407 #ifndef OPENSSL_NO_DH
408 /*
409  * PKCS #11 V2.20, section 11.2 specifies that the number of bytes needed for
410  * output buffer may somewhat exceed the precise number of bytes needed, but
411  * should not exceed it by a large amount. That may be caused, for example, by
412  * rounding it up to multiple of X in the underlying bignum library. 8 should be
413  * enough.
414  */
415 #define	DH_BUF_RESERVE	8
416 
417 /* Our internal DH_METHOD that we provide pointers to */
418 static DH_METHOD pk11_dh =
419 	{
420 	"PKCS#11 DH method",
421 	pk11_DH_generate_key,	/* generate_key */
422 	pk11_DH_compute_key,	/* compute_key */
423 	NULL,			/* bn_mod_exp */
424 	pk11_DH_init,		/* init */
425 	pk11_DH_finish,		/* finish */
426 	0,			/* flags */
427 	NULL,			/* app_data */
428 	NULL			/* generate_params */
429 	};
430 
431 DH_METHOD *
PK11_DH(void)432 PK11_DH(void)
433 	{
434 	return (&pk11_dh);
435 	}
436 #endif
437 
438 /* Size of an SSL signature: MD5+SHA1 */
439 #define	SSL_SIG_LENGTH		36
440 
441 /* Lengths of DSA data and signature */
442 #define	DSA_DATA_LEN		20
443 #define	DSA_SIGNATURE_LEN	40
444 
445 static CK_BBOOL true = TRUE;
446 static CK_BBOOL false = FALSE;
447 
448 #ifndef OPENSSL_NO_RSA
449 /*
450  * Similiar to OpenSSL to take advantage of the paddings. The goal is to
451  * support all paddings in this engine although PK11 library does not
452  * support all the paddings used in OpenSSL.
453  * The input errors should have been checked in the padding functions.
454  */
pk11_RSA_public_encrypt(int flen,const unsigned char * from,unsigned char * to,RSA * rsa,int padding)455 static int pk11_RSA_public_encrypt(int flen, const unsigned char *from,
456 		unsigned char *to, RSA *rsa, int padding)
457 	{
458 	int i, num = 0, r = -1;
459 	unsigned char *buf = NULL;
460 
461 	num = BN_num_bytes(rsa->n);
462 	if ((buf = (unsigned char *)OPENSSL_malloc(num)) == NULL)
463 		{
464 		RSAerr(PK11_F_RSA_PUB_ENC, PK11_R_MALLOC_FAILURE);
465 		goto err;
466 		}
467 
468 	switch (padding)
469 		{
470 	case RSA_PKCS1_PADDING:
471 		i = RSA_padding_add_PKCS1_type_2(buf, num, from, flen);
472 		break;
473 #ifndef OPENSSL_NO_SHA
474 	case RSA_PKCS1_OAEP_PADDING:
475 		i = RSA_padding_add_PKCS1_OAEP(buf, num, from, flen, NULL, 0);
476 		break;
477 #endif
478 	case RSA_SSLV23_PADDING:
479 		i = RSA_padding_add_SSLv23(buf, num, from, flen);
480 		break;
481 	case RSA_NO_PADDING:
482 		i = RSA_padding_add_none(buf, num, from, flen);
483 		break;
484 	default:
485 		RSAerr(PK11_F_RSA_PUB_ENC, PK11_R_UNKNOWN_PADDING_TYPE);
486 		goto err;
487 		}
488 	if (i <= 0) goto err;
489 
490 	/* PK11 functions are called here */
491 	r = pk11_RSA_public_encrypt_low(num, buf, to, rsa);
492 err:
493 	if (buf != NULL)
494 		{
495 		OPENSSL_cleanse(buf, num);
496 		OPENSSL_free(buf);
497 		}
498 	return (r);
499 	}
500 
501 
502 /*
503  * Similar to Openssl to take advantage of the paddings. The input errors
504  * should be catched in the padding functions
505  */
pk11_RSA_private_encrypt(int flen,const unsigned char * from,unsigned char * to,RSA * rsa,int padding)506 static int pk11_RSA_private_encrypt(int flen, const unsigned char *from,
507 	unsigned char *to, RSA *rsa, int padding)
508 	{
509 	int i, num = 0, r = -1;
510 	unsigned char *buf = NULL;
511 
512 	num = BN_num_bytes(rsa->n);
513 	if ((buf = (unsigned char *)OPENSSL_malloc(num)) == NULL)
514 		{
515 		RSAerr(PK11_F_RSA_PRIV_ENC, PK11_R_MALLOC_FAILURE);
516 		goto err;
517 		}
518 
519 	switch (padding)
520 		{
521 	case RSA_PKCS1_PADDING:
522 		i = RSA_padding_add_PKCS1_type_1(buf, num, from, flen);
523 		break;
524 	case RSA_NO_PADDING:
525 		i = RSA_padding_add_none(buf, num, from, flen);
526 		break;
527 	case RSA_SSLV23_PADDING:
528 	default:
529 		RSAerr(PK11_F_RSA_PRIV_ENC, PK11_R_UNKNOWN_PADDING_TYPE);
530 		goto err;
531 		}
532 	if (i <= 0) goto err;
533 
534 	/* PK11 functions are called here */
535 	r = pk11_RSA_private_encrypt_low(num, buf, to, rsa);
536 err:
537 	if (buf != NULL)
538 		{
539 		OPENSSL_cleanse(buf, num);
540 		OPENSSL_free(buf);
541 		}
542 	return (r);
543 	}
544 
545 /* Similar to OpenSSL code. Input errors are also checked here */
pk11_RSA_private_decrypt(int flen,const unsigned char * from,unsigned char * to,RSA * rsa,int padding)546 static int pk11_RSA_private_decrypt(int flen, const unsigned char *from,
547 	unsigned char *to, RSA *rsa, int padding)
548 	{
549 	BIGNUM f;
550 	int j, num = 0, r = -1;
551 	unsigned char *p;
552 	unsigned char *buf = NULL;
553 
554 	BN_init(&f);
555 
556 	num = BN_num_bytes(rsa->n);
557 
558 	if ((buf = (unsigned char *)OPENSSL_malloc(num)) == NULL)
559 		{
560 		RSAerr(PK11_F_RSA_PRIV_DEC, PK11_R_MALLOC_FAILURE);
561 		goto err;
562 		}
563 
564 	/*
565 	 * This check was for equality but PGP does evil things
566 	 * and chops off the top '0' bytes
567 	 */
568 	if (flen > num)
569 		{
570 		RSAerr(PK11_F_RSA_PRIV_DEC,
571 			PK11_R_DATA_GREATER_THAN_MOD_LEN);
572 		goto err;
573 		}
574 
575 	/* make data into a big number */
576 	if (BN_bin2bn(from, (int)flen, &f) == NULL)
577 		goto err;
578 
579 	if (BN_ucmp(&f, rsa->n) >= 0)
580 		{
581 		RSAerr(PK11_F_RSA_PRIV_DEC,
582 			PK11_R_DATA_TOO_LARGE_FOR_MODULUS);
583 		goto err;
584 		}
585 
586 	/* PK11 functions are called here */
587 	r = pk11_RSA_private_decrypt_low(flen, from, buf, rsa);
588 
589 	/*
590 	 * PK11 CKM_RSA_X_509 mechanism pads 0's at the beginning.
591 	 * Needs to skip these 0's paddings here.
592 	 */
593 	for (j = 0; j < r; j++)
594 		if (buf[j] != 0)
595 			break;
596 
597 	p = buf + j;
598 	j = r - j;  /* j is only used with no-padding mode */
599 
600 	switch (padding)
601 		{
602 	case RSA_PKCS1_PADDING:
603 		r = RSA_padding_check_PKCS1_type_2(to, num, p, j, num);
604 		break;
605 #ifndef OPENSSL_NO_SHA
606 	case RSA_PKCS1_OAEP_PADDING:
607 		r = RSA_padding_check_PKCS1_OAEP(to, num, p, j, num, NULL, 0);
608 		break;
609 #endif
610 	case RSA_SSLV23_PADDING:
611 		r = RSA_padding_check_SSLv23(to, num, p, j, num);
612 		break;
613 	case RSA_NO_PADDING:
614 		r = RSA_padding_check_none(to, num, p, j, num);
615 		break;
616 	default:
617 		RSAerr(PK11_F_RSA_PRIV_DEC, PK11_R_UNKNOWN_PADDING_TYPE);
618 		goto err;
619 		}
620 	if (r < 0)
621 		RSAerr(PK11_F_RSA_PRIV_DEC, PK11_R_PADDING_CHECK_FAILED);
622 
623 err:
624 	BN_clear_free(&f);
625 	if (buf != NULL)
626 		{
627 		OPENSSL_cleanse(buf, num);
628 		OPENSSL_free(buf);
629 		}
630 	return (r);
631 	}
632 
633 /* Similar to OpenSSL code. Input errors are also checked here */
pk11_RSA_public_decrypt(int flen,const unsigned char * from,unsigned char * to,RSA * rsa,int padding)634 static int pk11_RSA_public_decrypt(int flen, const unsigned char *from,
635 	unsigned char *to, RSA *rsa, int padding)
636 	{
637 	BIGNUM f;
638 	int i, num = 0, r = -1;
639 	unsigned char *p;
640 	unsigned char *buf = NULL;
641 
642 	BN_init(&f);
643 	num = BN_num_bytes(rsa->n);
644 	buf = (unsigned char *)OPENSSL_malloc(num);
645 	if (buf == NULL)
646 		{
647 		RSAerr(PK11_F_RSA_PUB_DEC, PK11_R_MALLOC_FAILURE);
648 		goto err;
649 		}
650 
651 	/*
652 	 * This check was for equality but PGP does evil things
653 	 * and chops off the top '0' bytes
654 	 */
655 	if (flen > num)
656 		{
657 		RSAerr(PK11_F_RSA_PUB_DEC, PK11_R_DATA_GREATER_THAN_MOD_LEN);
658 		goto err;
659 		}
660 
661 	if (BN_bin2bn(from, flen, &f) == NULL)
662 		goto err;
663 
664 	if (BN_ucmp(&f, rsa->n) >= 0)
665 		{
666 		RSAerr(PK11_F_RSA_PUB_DEC,
667 			PK11_R_DATA_TOO_LARGE_FOR_MODULUS);
668 		goto err;
669 		}
670 
671 	/* PK11 functions are called here */
672 	r = pk11_RSA_public_decrypt_low(flen, from, buf, rsa);
673 
674 	/*
675 	 * PK11 CKM_RSA_X_509 mechanism pads 0's at the beginning.
676 	 * Needs to skip these 0's here
677 	 */
678 	for (i = 0; i < r; i++)
679 		if (buf[i] != 0)
680 			break;
681 
682 	p = buf + i;
683 	i = r - i;  /* i is only used with no-padding mode */
684 
685 	switch (padding)
686 		{
687 	case RSA_PKCS1_PADDING:
688 		r = RSA_padding_check_PKCS1_type_1(to, num, p, i, num);
689 		break;
690 	case RSA_NO_PADDING:
691 		r = RSA_padding_check_none(to, num, p, i, num);
692 		break;
693 	default:
694 		RSAerr(PK11_F_RSA_PUB_DEC, PK11_R_UNKNOWN_PADDING_TYPE);
695 		goto err;
696 		}
697 	if (r < 0)
698 		RSAerr(PK11_F_RSA_PUB_DEC, PK11_R_PADDING_CHECK_FAILED);
699 
700 err:
701 	BN_clear_free(&f);
702 	if (buf != NULL)
703 		{
704 		OPENSSL_cleanse(buf, num);
705 		OPENSSL_free(buf);
706 		}
707 	return (r);
708 	}
709 
710 /*
711  * This function implements RSA public encryption using C_EncryptInit and
712  * C_Encrypt pk11 interfaces. Note that the CKM_RSA_X_509 is used here.
713  * The calling function allocated sufficient memory in "to" to store results.
714  */
pk11_RSA_public_encrypt_low(int flen,const unsigned char * from,unsigned char * to,RSA * rsa)715 static int pk11_RSA_public_encrypt_low(int flen,
716 	const unsigned char *from, unsigned char *to, RSA *rsa)
717 	{
718 	CK_ULONG bytes_encrypted = flen;
719 	int retval = -1;
720 	CK_RV rv;
721 	CK_MECHANISM mech_rsa = {CKM_RSA_X_509, NULL, 0};
722 	CK_MECHANISM *p_mech = &mech_rsa;
723 	CK_OBJECT_HANDLE h_pub_key = CK_INVALID_HANDLE;
724 	PK11_SESSION *sp;
725 
726 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
727 		return (-1);
728 
729 	(void) check_new_rsa_key_pub(sp, rsa);
730 
731 	h_pub_key = sp->opdata_rsa_pub_key;
732 	if (h_pub_key == CK_INVALID_HANDLE)
733 		h_pub_key = sp->opdata_rsa_pub_key =
734 			pk11_get_public_rsa_key(rsa, &sp->opdata_rsa_pub,
735 			    &sp->opdata_rsa_n_num, &sp->opdata_rsa_e_num,
736 			    sp->session);
737 
738 	if (h_pub_key != CK_INVALID_HANDLE)
739 		{
740 		rv = pFuncList->C_EncryptInit(sp->session, p_mech,
741 			h_pub_key);
742 
743 		if (rv != CKR_OK)
744 			{
745 			PK11err_add_data(PK11_F_RSA_PUB_ENC_LOW,
746 			    PK11_R_ENCRYPTINIT, rv);
747 			pk11_return_session(sp, OP_RSA);
748 			return (-1);
749 			}
750 
751 		rv = pFuncList->C_Encrypt(sp->session,
752 			(unsigned char *)from, flen, to, &bytes_encrypted);
753 
754 		if (rv != CKR_OK)
755 			{
756 			PK11err_add_data(PK11_F_RSA_PUB_ENC_LOW,
757 			    PK11_R_ENCRYPT, rv);
758 			pk11_return_session(sp, OP_RSA);
759 			return (-1);
760 			}
761 		retval = bytes_encrypted;
762 		}
763 
764 	pk11_return_session(sp, OP_RSA);
765 	return (retval);
766 	}
767 
768 
769 /*
770  * This function implements RSA private encryption using C_SignInit and
771  * C_Sign pk11 APIs. Note that CKM_RSA_X_509 is used here.
772  * The calling function allocated sufficient memory in "to" to store results.
773  */
pk11_RSA_private_encrypt_low(int flen,const unsigned char * from,unsigned char * to,RSA * rsa)774 static int pk11_RSA_private_encrypt_low(int flen,
775 	const unsigned char *from, unsigned char *to, RSA *rsa)
776 	{
777 	CK_ULONG ul_sig_len = flen;
778 	int retval = -1;
779 	CK_RV rv;
780 	CK_MECHANISM mech_rsa = {CKM_RSA_X_509, NULL, 0};
781 	CK_MECHANISM *p_mech = &mech_rsa;
782 	CK_OBJECT_HANDLE h_priv_key = CK_INVALID_HANDLE;
783 	PK11_SESSION *sp;
784 
785 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
786 		return (-1);
787 
788 	(void) check_new_rsa_key_priv(sp, rsa);
789 
790 	h_priv_key = sp->opdata_rsa_priv_key;
791 	if (h_priv_key == CK_INVALID_HANDLE)
792 		h_priv_key = sp->opdata_rsa_priv_key =
793 			pk11_get_private_rsa_key(rsa, &sp->opdata_rsa_priv,
794 			    &sp->opdata_rsa_d_num, sp->session);
795 
796 	if (h_priv_key != CK_INVALID_HANDLE)
797 		{
798 		rv = pFuncList->C_SignInit(sp->session, p_mech,
799 			h_priv_key);
800 
801 		if (rv != CKR_OK)
802 			{
803 			PK11err_add_data(PK11_F_RSA_PRIV_ENC_LOW,
804 			    PK11_R_SIGNINIT, rv);
805 			pk11_return_session(sp, OP_RSA);
806 			return (-1);
807 			}
808 
809 		rv = pFuncList->C_Sign(sp->session,
810 			(unsigned char *)from, flen, to, &ul_sig_len);
811 
812 		if (rv != CKR_OK)
813 			{
814 			PK11err_add_data(PK11_F_RSA_PRIV_ENC_LOW, PK11_R_SIGN,
815 			    rv);
816 			pk11_return_session(sp, OP_RSA);
817 			return (-1);
818 			}
819 
820 		retval = ul_sig_len;
821 		}
822 
823 	pk11_return_session(sp, OP_RSA);
824 	return (retval);
825 	}
826 
827 
828 /*
829  * This function implements RSA private decryption using C_DecryptInit and
830  * C_Decrypt pk11 APIs. Note that CKM_RSA_X_509 mechanism is used here.
831  * The calling function allocated sufficient memory in "to" to store results.
832  */
pk11_RSA_private_decrypt_low(int flen,const unsigned char * from,unsigned char * to,RSA * rsa)833 static int pk11_RSA_private_decrypt_low(int flen,
834 	const unsigned char *from, unsigned char *to, RSA *rsa)
835 	{
836 	CK_ULONG bytes_decrypted = flen;
837 	int retval = -1;
838 	CK_RV rv;
839 	CK_MECHANISM mech_rsa = {CKM_RSA_X_509, NULL, 0};
840 	CK_MECHANISM *p_mech = &mech_rsa;
841 	CK_OBJECT_HANDLE h_priv_key;
842 	PK11_SESSION *sp;
843 
844 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
845 		return (-1);
846 
847 	(void) check_new_rsa_key_priv(sp, rsa);
848 
849 	h_priv_key = sp->opdata_rsa_priv_key;
850 	if (h_priv_key == CK_INVALID_HANDLE)
851 		h_priv_key = sp->opdata_rsa_priv_key =
852 			pk11_get_private_rsa_key(rsa, &sp->opdata_rsa_priv,
853 			    &sp->opdata_rsa_d_num, sp->session);
854 
855 	if (h_priv_key != CK_INVALID_HANDLE)
856 		{
857 		rv = pFuncList->C_DecryptInit(sp->session, p_mech,
858 			h_priv_key);
859 
860 		if (rv != CKR_OK)
861 			{
862 			PK11err_add_data(PK11_F_RSA_PRIV_DEC_LOW,
863 				PK11_R_DECRYPTINIT, rv);
864 			pk11_return_session(sp, OP_RSA);
865 			return (-1);
866 			}
867 
868 		rv = pFuncList->C_Decrypt(sp->session,
869 			(unsigned char *)from, flen, to, &bytes_decrypted);
870 
871 		if (rv != CKR_OK)
872 			{
873 			PK11err_add_data(PK11_F_RSA_PRIV_DEC_LOW,
874 			    PK11_R_DECRYPT, rv);
875 			pk11_return_session(sp, OP_RSA);
876 			return (-1);
877 			}
878 		retval = bytes_decrypted;
879 		}
880 
881 	pk11_return_session(sp, OP_RSA);
882 	return (retval);
883 	}
884 
885 
886 /*
887  * This function implements RSA public decryption using C_VerifyRecoverInit
888  * and C_VerifyRecover pk11 APIs. Note that CKM_RSA_X_509 is used here.
889  * The calling function allocated sufficient memory in "to" to store results.
890  */
pk11_RSA_public_decrypt_low(int flen,const unsigned char * from,unsigned char * to,RSA * rsa)891 static int pk11_RSA_public_decrypt_low(int flen,
892 	const unsigned char *from, unsigned char *to, RSA *rsa)
893 	{
894 	CK_ULONG bytes_decrypted = flen;
895 	int retval = -1;
896 	CK_RV rv;
897 	CK_MECHANISM mech_rsa = {CKM_RSA_X_509, NULL, 0};
898 	CK_MECHANISM *p_mech = &mech_rsa;
899 	CK_OBJECT_HANDLE h_pub_key = CK_INVALID_HANDLE;
900 	PK11_SESSION *sp;
901 
902 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
903 		return (-1);
904 
905 	(void) check_new_rsa_key_pub(sp, rsa);
906 
907 	h_pub_key = sp->opdata_rsa_pub_key;
908 	if (h_pub_key == CK_INVALID_HANDLE)
909 		h_pub_key = sp->opdata_rsa_pub_key =
910 			pk11_get_public_rsa_key(rsa, &sp->opdata_rsa_pub,
911 			    &sp->opdata_rsa_n_num, &sp->opdata_rsa_e_num,
912 			    sp->session);
913 
914 	if (h_pub_key != CK_INVALID_HANDLE)
915 		{
916 		rv = pFuncList->C_VerifyRecoverInit(sp->session,
917 			p_mech, h_pub_key);
918 
919 		if (rv != CKR_OK)
920 			{
921 			PK11err_add_data(PK11_F_RSA_PUB_DEC_LOW,
922 				PK11_R_VERIFYRECOVERINIT, rv);
923 			pk11_return_session(sp, OP_RSA);
924 			return (-1);
925 			}
926 
927 		rv = pFuncList->C_VerifyRecover(sp->session,
928 			(unsigned char *)from, flen, to, &bytes_decrypted);
929 
930 		if (rv != CKR_OK)
931 			{
932 			PK11err_add_data(PK11_F_RSA_PUB_DEC_LOW,
933 			    PK11_R_VERIFYRECOVER, rv);
934 			pk11_return_session(sp, OP_RSA);
935 			return (-1);
936 			}
937 		retval = bytes_decrypted;
938 		}
939 
940 	pk11_return_session(sp, OP_RSA);
941 	return (retval);
942 	}
943 
pk11_RSA_init(RSA * rsa)944 static int pk11_RSA_init(RSA *rsa)
945 	{
946 	/*
947 	 * This flag in the RSA_METHOD enables the new rsa_sign,
948 	 * rsa_verify functions. See rsa.h for details.
949 	 */
950 	rsa->flags |= RSA_FLAG_SIGN_VER;
951 
952 	return (1);
953 	}
954 
pk11_RSA_finish(RSA * rsa)955 static int pk11_RSA_finish(RSA *rsa)
956 	{
957 	/*
958 	 * Since we are overloading OpenSSL's native RSA_eay_finish() we need
959 	 * to do the same as in the original function, i.e. to free bignum
960 	 * structures.
961 	 */
962 	if (rsa->_method_mod_n != NULL)
963 		BN_MONT_CTX_free(rsa->_method_mod_n);
964 	if (rsa->_method_mod_p != NULL)
965 		BN_MONT_CTX_free(rsa->_method_mod_p);
966 	if (rsa->_method_mod_q != NULL)
967 		BN_MONT_CTX_free(rsa->_method_mod_q);
968 
969 	return (1);
970 	}
971 
972 /*
973  * Standard engine interface function. Majority codes here are from
974  * rsa/rsa_sign.c. We replaced the decrypt function call by C_Sign of PKCS#11.
975  * See more details in rsa/rsa_sign.c
976  */
pk11_RSA_sign(int type,const unsigned char * m,unsigned int m_len,unsigned char * sigret,unsigned int * siglen,const RSA * rsa)977 static int pk11_RSA_sign(int type, const unsigned char *m, unsigned int m_len,
978 	unsigned char *sigret, unsigned int *siglen, const RSA *rsa)
979 	{
980 	X509_SIG sig;
981 	ASN1_TYPE parameter;
982 	int i, j;
983 	unsigned char *p, *s = NULL;
984 	X509_ALGOR algor;
985 	ASN1_OCTET_STRING digest;
986 	CK_RV rv;
987 	CK_MECHANISM mech_rsa = {CKM_RSA_PKCS, NULL, 0};
988 	CK_MECHANISM *p_mech = &mech_rsa;
989 	CK_OBJECT_HANDLE h_priv_key;
990 	PK11_SESSION *sp = NULL;
991 	int ret = 0;
992 	unsigned long ulsiglen;
993 
994 	/* Encode the digest */
995 	/* Special case: SSL signature, just check the length */
996 	if (type == NID_md5_sha1)
997 		{
998 		if (m_len != SSL_SIG_LENGTH)
999 			{
1000 			PK11err(PK11_F_RSA_SIGN,
1001 				PK11_R_INVALID_MESSAGE_LENGTH);
1002 			goto err;
1003 			}
1004 		i = SSL_SIG_LENGTH;
1005 		s = (unsigned char *)m;
1006 		}
1007 	else
1008 		{
1009 		sig.algor = &algor;
1010 		sig.algor->algorithm = OBJ_nid2obj(type);
1011 		if (sig.algor->algorithm == NULL)
1012 			{
1013 			PK11err(PK11_F_RSA_SIGN,
1014 				PK11_R_UNKNOWN_ALGORITHM_TYPE);
1015 			goto err;
1016 			}
1017 		if (sig.algor->algorithm->length == 0)
1018 			{
1019 			PK11err(PK11_F_RSA_SIGN,
1020 				PK11_R_UNKNOWN_ASN1_OBJECT_ID);
1021 			goto err;
1022 			}
1023 		parameter.type = V_ASN1_NULL;
1024 		parameter.value.ptr = NULL;
1025 		sig.algor->parameter = &parameter;
1026 
1027 		sig.digest = &digest;
1028 		sig.digest->data = (unsigned char *)m;
1029 		sig.digest->length = m_len;
1030 
1031 		i = i2d_X509_SIG(&sig, NULL);
1032 		}
1033 
1034 	j = RSA_size(rsa);
1035 	if ((i - RSA_PKCS1_PADDING) > j)
1036 		{
1037 		PK11err(PK11_F_RSA_SIGN, PK11_R_DIGEST_TOO_BIG);
1038 		goto err;
1039 		}
1040 
1041 	if (type != NID_md5_sha1)
1042 		{
1043 		s = (unsigned char *)OPENSSL_malloc((unsigned int)(j + 1));
1044 		if (s == NULL)
1045 			{
1046 			PK11err(PK11_F_RSA_SIGN, PK11_R_MALLOC_FAILURE);
1047 			goto err;
1048 			}
1049 		p = s;
1050 		(void) i2d_X509_SIG(&sig, &p);
1051 		}
1052 
1053 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
1054 		goto err;
1055 
1056 	(void) check_new_rsa_key_priv(sp, rsa);
1057 
1058 	h_priv_key = sp->opdata_rsa_priv_key;
1059 	if (h_priv_key == CK_INVALID_HANDLE)
1060 		h_priv_key = sp->opdata_rsa_priv_key =
1061 			pk11_get_private_rsa_key((RSA *)rsa,
1062 			    &sp->opdata_rsa_priv,
1063 			    &sp->opdata_rsa_d_num, sp->session);
1064 
1065 	if (h_priv_key != CK_INVALID_HANDLE)
1066 		{
1067 		rv = pFuncList->C_SignInit(sp->session, p_mech, h_priv_key);
1068 
1069 		if (rv != CKR_OK)
1070 			{
1071 			PK11err_add_data(PK11_F_RSA_SIGN, PK11_R_SIGNINIT, rv);
1072 			goto err;
1073 			}
1074 
1075 		ulsiglen = j;
1076 		rv = pFuncList->C_Sign(sp->session, s, i, sigret,
1077 			(CK_ULONG_PTR) &ulsiglen);
1078 		*siglen = ulsiglen;
1079 
1080 		if (rv != CKR_OK)
1081 			{
1082 			PK11err_add_data(PK11_F_RSA_SIGN, PK11_R_SIGN, rv);
1083 			goto err;
1084 			}
1085 		ret = 1;
1086 		}
1087 
1088 err:
1089 	if (type != NID_md5_sha1)
1090 		{
1091 		(void) memset(s, 0, (unsigned int)(j + 1));
1092 		OPENSSL_free(s);
1093 		}
1094 
1095 	pk11_return_session(sp, OP_RSA);
1096 	return (ret);
1097 	}
1098 
pk11_RSA_verify(int type,const unsigned char * m,unsigned int m_len,unsigned char * sigbuf,unsigned int siglen,const RSA * rsa)1099 static int pk11_RSA_verify(int type, const unsigned char *m,
1100 	unsigned int m_len, unsigned char *sigbuf, unsigned int siglen,
1101 	const RSA *rsa)
1102 	{
1103 	X509_SIG sig;
1104 	ASN1_TYPE parameter;
1105 	int i, j;
1106 	unsigned char *p, *s = NULL;
1107 	X509_ALGOR algor;
1108 	ASN1_OCTET_STRING digest;
1109 	CK_RV rv;
1110 	CK_MECHANISM mech_rsa = {CKM_RSA_PKCS, NULL, 0};
1111 	CK_MECHANISM *p_mech = &mech_rsa;
1112 	CK_OBJECT_HANDLE h_pub_key;
1113 	PK11_SESSION *sp = NULL;
1114 	int ret = 0;
1115 
1116 	/* Encode the digest	*/
1117 	/* Special case: SSL signature, just check the length */
1118 	if (type == NID_md5_sha1)
1119 		{
1120 		if (m_len != SSL_SIG_LENGTH)
1121 			{
1122 			PK11err(PK11_F_RSA_VERIFY,
1123 				PK11_R_INVALID_MESSAGE_LENGTH);
1124 			goto err;
1125 			}
1126 		i = SSL_SIG_LENGTH;
1127 		s = (unsigned char *)m;
1128 		}
1129 	else
1130 		{
1131 		sig.algor = &algor;
1132 		sig.algor->algorithm = OBJ_nid2obj(type);
1133 		if (sig.algor->algorithm == NULL)
1134 			{
1135 			PK11err(PK11_F_RSA_VERIFY,
1136 				PK11_R_UNKNOWN_ALGORITHM_TYPE);
1137 			goto err;
1138 			}
1139 		if (sig.algor->algorithm->length == 0)
1140 			{
1141 			PK11err(PK11_F_RSA_VERIFY,
1142 				PK11_R_UNKNOWN_ASN1_OBJECT_ID);
1143 			goto err;
1144 			}
1145 		parameter.type = V_ASN1_NULL;
1146 		parameter.value.ptr = NULL;
1147 		sig.algor->parameter = &parameter;
1148 		sig.digest = &digest;
1149 		sig.digest->data = (unsigned char *)m;
1150 		sig.digest->length = m_len;
1151 		i = i2d_X509_SIG(&sig, NULL);
1152 		}
1153 
1154 	j = RSA_size(rsa);
1155 	if ((i - RSA_PKCS1_PADDING) > j)
1156 		{
1157 		PK11err(PK11_F_RSA_VERIFY, PK11_R_DIGEST_TOO_BIG);
1158 		goto err;
1159 		}
1160 
1161 	if (type != NID_md5_sha1)
1162 		{
1163 		s = (unsigned char *)OPENSSL_malloc((unsigned int)(j + 1));
1164 		if (s == NULL)
1165 			{
1166 			PK11err(PK11_F_RSA_VERIFY, PK11_R_MALLOC_FAILURE);
1167 			goto err;
1168 			}
1169 		p = s;
1170 		(void) i2d_X509_SIG(&sig, &p);
1171 		}
1172 
1173 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
1174 		goto err;
1175 
1176 	(void) check_new_rsa_key_pub(sp, rsa);
1177 
1178 	h_pub_key = sp->opdata_rsa_pub_key;
1179 	if (h_pub_key == CK_INVALID_HANDLE)
1180 		h_pub_key = sp->opdata_rsa_pub_key =
1181 			pk11_get_public_rsa_key((RSA *)rsa, &sp->opdata_rsa_pub,
1182 			    &sp->opdata_rsa_n_num, &sp->opdata_rsa_e_num,
1183 			    sp->session);
1184 
1185 	if (h_pub_key != CK_INVALID_HANDLE)
1186 		{
1187 		rv = pFuncList->C_VerifyInit(sp->session, p_mech,
1188 			h_pub_key);
1189 
1190 		if (rv != CKR_OK)
1191 			{
1192 			PK11err_add_data(PK11_F_RSA_VERIFY, PK11_R_VERIFYINIT,
1193 			    rv);
1194 			goto err;
1195 			}
1196 		rv = pFuncList->C_Verify(sp->session, s, i, sigbuf,
1197 			(CK_ULONG)siglen);
1198 
1199 		if (rv != CKR_OK)
1200 			{
1201 			PK11err_add_data(PK11_F_RSA_VERIFY, PK11_R_VERIFY, rv);
1202 			goto err;
1203 			}
1204 		ret = 1;
1205 		}
1206 
1207 err:
1208 	if (type != NID_md5_sha1)
1209 		{
1210 		(void) memset(s, 0, (unsigned int)siglen);
1211 		OPENSSL_free(s);
1212 		}
1213 
1214 	pk11_return_session(sp, OP_RSA);
1215 	return (ret);
1216 	}
1217 
1218 /* load RSA private key from a file */
1219 /* ARGSUSED */
pk11_load_privkey(ENGINE * e,const char * privkey_file,UI_METHOD * ui_method,void * callback_data)1220 EVP_PKEY *pk11_load_privkey(ENGINE* e, const char *privkey_file,
1221 	UI_METHOD *ui_method, void *callback_data)
1222 	{
1223 	EVP_PKEY *pkey = NULL;
1224 	FILE *pubkey;
1225 	CK_OBJECT_HANDLE  h_priv_key = CK_INVALID_HANDLE;
1226 	RSA *rsa;
1227 	PK11_SESSION *sp;
1228 
1229 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
1230 		return (NULL);
1231 
1232 	if ((pubkey = fopen(privkey_file, read_mode_flags)) != NULL)
1233 		{
1234 		pkey = PEM_read_PrivateKey(pubkey, NULL, NULL, NULL);
1235 		(void) fclose(pubkey);
1236 		if (pkey != NULL)
1237 			{
1238 			rsa = EVP_PKEY_get1_RSA(pkey);
1239 			if (rsa != NULL)
1240 				{
1241 				(void) check_new_rsa_key_priv(sp, rsa);
1242 
1243 				h_priv_key = sp->opdata_rsa_priv_key =
1244 				    pk11_get_private_rsa_key(rsa,
1245 				    &sp->opdata_rsa_priv, &sp->opdata_rsa_d_num,
1246 				    sp->session);
1247 				if (h_priv_key == CK_INVALID_HANDLE)
1248 					{
1249 					EVP_PKEY_free(pkey);
1250 					pkey = NULL;
1251 					}
1252 				}
1253 			else
1254 				{
1255 				EVP_PKEY_free(pkey);
1256 				pkey = NULL;
1257 				}
1258 			}
1259 		}
1260 
1261 	pk11_return_session(sp, OP_RSA);
1262 	return (pkey);
1263 	}
1264 
1265 /* load RSA public key from a file */
1266 /* ARGSUSED */
pk11_load_pubkey(ENGINE * e,const char * pubkey_file,UI_METHOD * ui_method,void * callback_data)1267 EVP_PKEY *pk11_load_pubkey(ENGINE* e, const char *pubkey_file,
1268 	UI_METHOD *ui_method, void *callback_data)
1269 	{
1270 	EVP_PKEY *pkey = NULL;
1271 	FILE *pubkey;
1272 	CK_OBJECT_HANDLE  h_pub_key = CK_INVALID_HANDLE;
1273 	RSA *rsa;
1274 	PK11_SESSION *sp;
1275 
1276 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
1277 		return (NULL);
1278 
1279 	if ((pubkey = fopen(pubkey_file, read_mode_flags)) != NULL)
1280 		{
1281 		pkey = PEM_read_PUBKEY(pubkey, NULL, NULL, NULL);
1282 		(void) fclose(pubkey);
1283 		if (pkey != NULL)
1284 			{
1285 			rsa = EVP_PKEY_get1_RSA(pkey);
1286 			if (rsa != NULL)
1287 				{
1288 				(void) check_new_rsa_key_pub(sp, rsa);
1289 
1290 				h_pub_key = sp->opdata_rsa_pub_key =
1291 				    pk11_get_public_rsa_key(rsa,
1292 				    &sp->opdata_rsa_pub, &sp->opdata_rsa_n_num,
1293 				    &sp->opdata_rsa_e_num, sp->session);
1294 				if (h_pub_key == CK_INVALID_HANDLE)
1295 					{
1296 					EVP_PKEY_free(pkey);
1297 					pkey = NULL;
1298 					}
1299 				}
1300 			else
1301 				{
1302 				EVP_PKEY_free(pkey);
1303 				pkey = NULL;
1304 				}
1305 			}
1306 		}
1307 
1308 	pk11_return_session(sp, OP_RSA);
1309 	return (pkey);
1310 	}
1311 
1312 /*
1313  * Create a public key object in a session from a given rsa structure.
1314  * The *rsa_n_num and *rsa_e_num pointers are non-NULL for RSA public keys.
1315  */
pk11_get_public_rsa_key(RSA * rsa,RSA ** key_ptr,BIGNUM ** rsa_n_num,BIGNUM ** rsa_e_num,CK_SESSION_HANDLE session)1316 static CK_OBJECT_HANDLE pk11_get_public_rsa_key(RSA* rsa,
1317     RSA** key_ptr, BIGNUM **rsa_n_num, BIGNUM **rsa_e_num,
1318     CK_SESSION_HANDLE session)
1319 	{
1320 	CK_RV rv;
1321 	CK_OBJECT_HANDLE h_key = CK_INVALID_HANDLE;
1322 	CK_ULONG found;
1323 	CK_OBJECT_CLASS o_key = CKO_PUBLIC_KEY;
1324 	CK_KEY_TYPE k_type = CKK_RSA;
1325 	CK_ULONG ul_key_attr_count = 7;
1326 	CK_BBOOL rollback = FALSE;
1327 
1328 	CK_ATTRIBUTE  a_key_template[] =
1329 		{
1330 		{CKA_CLASS, (void *) NULL, sizeof (CK_OBJECT_CLASS)},
1331 		{CKA_KEY_TYPE, (void *) NULL, sizeof (CK_KEY_TYPE)},
1332 		{CKA_TOKEN, &false, sizeof (true)},
1333 		{CKA_ENCRYPT, &true, sizeof (true)},
1334 		{CKA_VERIFY_RECOVER, &true, sizeof (true)},
1335 		{CKA_MODULUS, (void *)NULL, 0},
1336 		{CKA_PUBLIC_EXPONENT, (void *)NULL, 0}
1337 		};
1338 
1339 	int i;
1340 
1341 	a_key_template[0].pValue = &o_key;
1342 	a_key_template[1].pValue = &k_type;
1343 
1344 	a_key_template[5].ulValueLen = BN_num_bytes(rsa->n);
1345 	a_key_template[5].pValue = (CK_VOID_PTR)OPENSSL_malloc(
1346 		(size_t)a_key_template[5].ulValueLen);
1347 	if (a_key_template[5].pValue == NULL)
1348 		{
1349 		PK11err(PK11_F_GET_PUB_RSA_KEY, PK11_R_MALLOC_FAILURE);
1350 		goto malloc_err;
1351 		}
1352 
1353 	(void) BN_bn2bin(rsa->n, a_key_template[5].pValue);
1354 
1355 	a_key_template[6].ulValueLen = BN_num_bytes(rsa->e);
1356 	a_key_template[6].pValue = (CK_VOID_PTR)OPENSSL_malloc(
1357 		(size_t)a_key_template[6].ulValueLen);
1358 	if (a_key_template[6].pValue == NULL)
1359 		{
1360 		PK11err(PK11_F_GET_PUB_RSA_KEY, PK11_R_MALLOC_FAILURE);
1361 		goto malloc_err;
1362 		}
1363 
1364 	(void) BN_bn2bin(rsa->e, a_key_template[6].pValue);
1365 
1366 	/* see find_lock array definition for more info on object locking */
1367 	LOCK_OBJSTORE(OP_RSA);
1368 	rv = pFuncList->C_FindObjectsInit(session, a_key_template,
1369 		ul_key_attr_count);
1370 
1371 	if (rv != CKR_OK)
1372 		{
1373 		PK11err_add_data(PK11_F_GET_PUB_RSA_KEY, PK11_R_FINDOBJECTSINIT,
1374 		    rv);
1375 		goto err;
1376 		}
1377 
1378 	rv = pFuncList->C_FindObjects(session, &h_key, 1, &found);
1379 
1380 	if (rv != CKR_OK)
1381 		{
1382 		PK11err_add_data(PK11_F_GET_PUB_RSA_KEY,
1383 		    PK11_R_FINDOBJECTS, rv);
1384 		goto err;
1385 		}
1386 
1387 	rv = pFuncList->C_FindObjectsFinal(session);
1388 
1389 	if (rv != CKR_OK)
1390 		{
1391 		PK11err_add_data(PK11_F_GET_PUB_RSA_KEY,
1392 		    PK11_R_FINDOBJECTSFINAL, rv);
1393 		goto err;
1394 		}
1395 
1396 	if (found == 0)
1397 		{
1398 		rv = pFuncList->C_CreateObject(session,
1399 			a_key_template, ul_key_attr_count, &h_key);
1400 		if (rv != CKR_OK)
1401 			{
1402 			PK11err_add_data(PK11_F_GET_PUB_RSA_KEY,
1403 			    PK11_R_CREATEOBJECT, rv);
1404 			goto err;
1405 			}
1406 		}
1407 
1408 	if (rsa_n_num != NULL)
1409 		if ((*rsa_n_num = BN_dup(rsa->n)) == NULL)
1410 			{
1411 			PK11err(PK11_F_GET_PUB_RSA_KEY, PK11_R_MALLOC_FAILURE);
1412 			rollback = TRUE;
1413 			goto err;
1414 			}
1415 	if (rsa_e_num != NULL)
1416 		if ((*rsa_e_num = BN_dup(rsa->e)) == NULL)
1417 			{
1418 			PK11err(PK11_F_GET_PUB_RSA_KEY, PK11_R_MALLOC_FAILURE);
1419 			BN_free(*rsa_n_num);
1420 			*rsa_n_num = NULL;
1421 			rollback = TRUE;
1422 			goto err;
1423 			}
1424 
1425 	/* LINTED: E_CONSTANT_CONDITION */
1426 	KEY_HANDLE_REFHOLD(h_key, OP_RSA, FALSE, rollback, err);
1427 	if (key_ptr != NULL)
1428 		*key_ptr = rsa;
1429 
1430 err:
1431 	if (rollback)
1432 		{
1433 		/*
1434 		 * We do not care about the return value from C_DestroyObject()
1435 		 * since we are doing rollback.
1436 		 */
1437 		if (found == 0)
1438 			(void) pFuncList->C_DestroyObject(session, h_key);
1439 		h_key = CK_INVALID_HANDLE;
1440 		}
1441 
1442 	UNLOCK_OBJSTORE(OP_RSA);
1443 
1444 malloc_err:
1445 	for (i = 5; i <= 6; i++)
1446 		{
1447 		if (a_key_template[i].pValue != NULL)
1448 			{
1449 			OPENSSL_free(a_key_template[i].pValue);
1450 			a_key_template[i].pValue = NULL;
1451 			}
1452 		}
1453 
1454 	return (h_key);
1455 	}
1456 
1457 /*
1458  * Create a private key object in the session from a given rsa structure.
1459  * The *rsa_d_num pointer is non-NULL for RSA private keys.
1460  */
pk11_get_private_rsa_key(RSA * rsa,RSA ** key_ptr,BIGNUM ** rsa_d_num,CK_SESSION_HANDLE session)1461 static CK_OBJECT_HANDLE pk11_get_private_rsa_key(RSA* rsa,
1462     RSA** key_ptr, BIGNUM **rsa_d_num, CK_SESSION_HANDLE session)
1463 	{
1464 	CK_RV rv;
1465 	CK_OBJECT_HANDLE h_key = CK_INVALID_HANDLE;
1466 	int i;
1467 	CK_ULONG found;
1468 	CK_OBJECT_CLASS o_key = CKO_PRIVATE_KEY;
1469 	CK_KEY_TYPE k_type = CKK_RSA;
1470 	CK_ULONG ul_key_attr_count = 14;
1471 	CK_BBOOL rollback = FALSE;
1472 
1473 	/* Both CKA_TOKEN and CKA_SENSITIVE have to be FALSE for session keys */
1474 	CK_ATTRIBUTE  a_key_template[] =
1475 		{
1476 		{CKA_CLASS, (void *) NULL, sizeof (CK_OBJECT_CLASS)},
1477 		{CKA_KEY_TYPE, (void *) NULL, sizeof (CK_KEY_TYPE)},
1478 		{CKA_TOKEN, &false, sizeof (true)},
1479 		{CKA_SENSITIVE, &false, sizeof (true)},
1480 		{CKA_DECRYPT, &true, sizeof (true)},
1481 		{CKA_SIGN, &true, sizeof (true)},
1482 		{CKA_MODULUS, (void *)NULL, 0},
1483 		{CKA_PUBLIC_EXPONENT, (void *)NULL, 0},
1484 		{CKA_PRIVATE_EXPONENT, (void *)NULL, 0},
1485 		{CKA_PRIME_1, (void *)NULL, 0},
1486 		{CKA_PRIME_2, (void *)NULL, 0},
1487 		{CKA_EXPONENT_1, (void *)NULL, 0},
1488 		{CKA_EXPONENT_2, (void *)NULL, 0},
1489 		{CKA_COEFFICIENT, (void *)NULL, 0}
1490 		};
1491 
1492 	a_key_template[0].pValue = &o_key;
1493 	a_key_template[1].pValue = &k_type;
1494 
1495 	/* Put the private key components into the template */
1496 	if (init_template_value(rsa->n, &a_key_template[6].pValue,
1497 		&a_key_template[6].ulValueLen) == 0 ||
1498 	    init_template_value(rsa->e, &a_key_template[7].pValue,
1499 		&a_key_template[7].ulValueLen) == 0 ||
1500 	    init_template_value(rsa->d, &a_key_template[8].pValue,
1501 		&a_key_template[8].ulValueLen) == 0 ||
1502 	    init_template_value(rsa->p, &a_key_template[9].pValue,
1503 		&a_key_template[9].ulValueLen) == 0 ||
1504 	    init_template_value(rsa->q, &a_key_template[10].pValue,
1505 		&a_key_template[10].ulValueLen) == 0 ||
1506 	    init_template_value(rsa->dmp1, &a_key_template[11].pValue,
1507 		&a_key_template[11].ulValueLen) == 0 ||
1508 	    init_template_value(rsa->dmq1, &a_key_template[12].pValue,
1509 		&a_key_template[12].ulValueLen) == 0 ||
1510 	    init_template_value(rsa->iqmp, &a_key_template[13].pValue,
1511 		&a_key_template[13].ulValueLen) == 0)
1512 		{
1513 		PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
1514 		goto malloc_err;
1515 		}
1516 
1517 	/* see find_lock array definition for more info on object locking */
1518 	LOCK_OBJSTORE(OP_RSA);
1519 	rv = pFuncList->C_FindObjectsInit(session, a_key_template,
1520 		ul_key_attr_count);
1521 
1522 	if (rv != CKR_OK)
1523 		{
1524 		PK11err_add_data(PK11_F_GET_PRIV_RSA_KEY,
1525 		    PK11_R_FINDOBJECTSINIT, rv);
1526 		goto err;
1527 		}
1528 
1529 	rv = pFuncList->C_FindObjects(session, &h_key, 1, &found);
1530 
1531 	if (rv != CKR_OK)
1532 		{
1533 		PK11err_add_data(PK11_F_GET_PRIV_RSA_KEY,
1534 		    PK11_R_FINDOBJECTS, rv);
1535 		goto err;
1536 		}
1537 
1538 	rv = pFuncList->C_FindObjectsFinal(session);
1539 
1540 	if (rv != CKR_OK)
1541 		{
1542 		PK11err_add_data(PK11_F_GET_PRIV_RSA_KEY,
1543 		    PK11_R_FINDOBJECTSFINAL, rv);
1544 		goto err;
1545 		}
1546 
1547 	if (found == 0)
1548 		{
1549 		rv = pFuncList->C_CreateObject(session,
1550 			a_key_template, ul_key_attr_count, &h_key);
1551 		if (rv != CKR_OK)
1552 			{
1553 			PK11err_add_data(PK11_F_GET_PRIV_RSA_KEY,
1554 				PK11_R_CREATEOBJECT, rv);
1555 			goto err;
1556 			}
1557 		}
1558 
1559 	if (rsa_d_num != NULL)
1560 		if ((*rsa_d_num = BN_dup(rsa->d)) == NULL)
1561 			{
1562 			PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
1563 			rollback = TRUE;
1564 			goto err;
1565 			}
1566 
1567 	/* LINTED: E_CONSTANT_CONDITION */
1568 	KEY_HANDLE_REFHOLD(h_key, OP_RSA, FALSE, rollback, err);
1569 	if (key_ptr != NULL)
1570 		*key_ptr = rsa;
1571 
1572 err:
1573 	if (rollback)
1574 		{
1575 		/*
1576 		 * We do not care about the return value from C_DestroyObject()
1577 		 * since we are doing rollback.
1578 		 */
1579 		if (found == 0)
1580 			(void) pFuncList->C_DestroyObject(session, h_key);
1581 		h_key = CK_INVALID_HANDLE;
1582 		}
1583 
1584 	UNLOCK_OBJSTORE(OP_RSA);
1585 
1586 malloc_err:
1587 	/*
1588 	 * 6 to 13 entries in the key template are key components.
1589 	 * They need to be freed apon exit or error.
1590 	 */
1591 	for (i = 6; i <= 13; i++)
1592 		{
1593 		if (a_key_template[i].pValue != NULL)
1594 			{
1595 			(void) memset(a_key_template[i].pValue, 0,
1596 				a_key_template[i].ulValueLen);
1597 			OPENSSL_free(a_key_template[i].pValue);
1598 			a_key_template[i].pValue = NULL;
1599 			}
1600 		}
1601 
1602 	return (h_key);
1603 	}
1604 
1605 /*
1606  * Check for cache miss and clean the object pointer and handle
1607  * in such case. Return 1 for cache hit, 0 for cache miss.
1608  */
check_new_rsa_key_pub(PK11_SESSION * sp,const RSA * rsa)1609 static int check_new_rsa_key_pub(PK11_SESSION *sp, const RSA *rsa)
1610 	{
1611 	/*
1612 	 * Provide protection against RSA structure reuse by making the
1613 	 * check for cache hit stronger. Only public components of RSA
1614 	 * key matter here so it is sufficient to compare them with values
1615 	 * cached in PK11_SESSION structure.
1616 	 */
1617 	if ((sp->opdata_rsa_pub != rsa) ||
1618 	    (BN_cmp(sp->opdata_rsa_n_num, rsa->n) != 0) ||
1619 	    (BN_cmp(sp->opdata_rsa_e_num, rsa->e) != 0))
1620 		{
1621 		/*
1622 		 * We do not check the return value because even in case of
1623 		 * failure the sp structure will have both key pointer
1624 		 * and object handle cleaned and pk11_destroy_object()
1625 		 * reports the failure to the OpenSSL error message buffer.
1626 		 */
1627 		(void) pk11_destroy_rsa_object_pub(sp, TRUE);
1628 		return (0);
1629 		}
1630 	return (1);
1631 	}
1632 
1633 /*
1634  * Check for cache miss and clean the object pointer and handle
1635  * in such case. Return 1 for cache hit, 0 for cache miss.
1636  */
check_new_rsa_key_priv(PK11_SESSION * sp,const RSA * rsa)1637 static int check_new_rsa_key_priv(PK11_SESSION *sp, const RSA *rsa)
1638 	{
1639 	/*
1640 	 * Provide protection against RSA structure reuse by making the
1641 	 * check for cache hit stronger. Comparing private exponent of RSA
1642 	 * key with value cached in PK11_SESSION structure should
1643 	 * be sufficient.
1644 	 */
1645 	if ((sp->opdata_rsa_priv != rsa) ||
1646 	    (BN_cmp(sp->opdata_rsa_d_num, rsa->d) != 0))
1647 		{
1648 		/*
1649 		 * We do not check the return value because even in case of
1650 		 * failure the sp structure will have both key pointer
1651 		 * and object handle cleaned and pk11_destroy_object()
1652 		 * reports the failure to the OpenSSL error message buffer.
1653 		 */
1654 		(void) pk11_destroy_rsa_object_priv(sp, TRUE);
1655 		return (0);
1656 		}
1657 	return (1);
1658 	}
1659 #endif
1660 
1661 #ifndef OPENSSL_NO_DSA
1662 /* The DSA function implementation */
1663 /* ARGSUSED */
pk11_DSA_init(DSA * dsa)1664 static int pk11_DSA_init(DSA *dsa)
1665 	{
1666 	return (1);
1667 	}
1668 
1669 /* ARGSUSED */
pk11_DSA_finish(DSA * dsa)1670 static int pk11_DSA_finish(DSA *dsa)
1671 	{
1672 	return (1);
1673 	}
1674 
1675 
1676 static DSA_SIG *
pk11_dsa_do_sign(const unsigned char * dgst,int dlen,DSA * dsa)1677 pk11_dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa)
1678 	{
1679 	BIGNUM *r = NULL, *s = NULL;
1680 	int i;
1681 	DSA_SIG *dsa_sig = NULL;
1682 
1683 	CK_RV rv;
1684 	CK_MECHANISM Mechanism_dsa = {CKM_DSA, NULL, 0};
1685 	CK_MECHANISM *p_mech = &Mechanism_dsa;
1686 	CK_OBJECT_HANDLE h_priv_key;
1687 
1688 	/*
1689 	 * The signature is the concatenation of r and s,
1690 	 * each is 20 bytes long
1691 	 */
1692 	unsigned char sigret[DSA_SIGNATURE_LEN];
1693 	unsigned long siglen = DSA_SIGNATURE_LEN;
1694 	unsigned int siglen2 = DSA_SIGNATURE_LEN / 2;
1695 
1696 	PK11_SESSION *sp = NULL;
1697 
1698 	if ((dsa->p == NULL) || (dsa->q == NULL) || (dsa->g == NULL))
1699 		{
1700 		PK11err(PK11_F_DSA_SIGN, PK11_R_MISSING_KEY_COMPONENT);
1701 		goto ret;
1702 		}
1703 
1704 	i = BN_num_bytes(dsa->q); /* should be 20 */
1705 	if (dlen > i)
1706 		{
1707 		PK11err(PK11_F_DSA_SIGN, PK11_R_INVALID_SIGNATURE_LENGTH);
1708 		goto ret;
1709 		}
1710 
1711 	if ((sp = pk11_get_session(OP_DSA)) == NULL)
1712 		goto ret;
1713 
1714 	(void) check_new_dsa_key_priv(sp, dsa);
1715 
1716 	h_priv_key = sp->opdata_dsa_priv_key;
1717 	if (h_priv_key == CK_INVALID_HANDLE)
1718 		h_priv_key = sp->opdata_dsa_priv_key =
1719 			pk11_get_private_dsa_key((DSA *)dsa,
1720 			    &sp->opdata_dsa_priv,
1721 			    &sp->opdata_dsa_priv_num, sp->session);
1722 
1723 	if (h_priv_key != CK_INVALID_HANDLE)
1724 		{
1725 		rv = pFuncList->C_SignInit(sp->session, p_mech, h_priv_key);
1726 
1727 		if (rv != CKR_OK)
1728 			{
1729 			PK11err_add_data(PK11_F_DSA_SIGN, PK11_R_SIGNINIT, rv);
1730 			goto ret;
1731 			}
1732 
1733 			(void) memset(sigret, 0, siglen);
1734 			rv = pFuncList->C_Sign(sp->session,
1735 			    (unsigned char *) dgst, dlen, sigret,
1736 			    (CK_ULONG_PTR) &siglen);
1737 
1738 		if (rv != CKR_OK)
1739 			{
1740 			PK11err_add_data(PK11_F_DSA_SIGN, PK11_R_SIGN, rv);
1741 			goto ret;
1742 			}
1743 		}
1744 
1745 
1746 	if ((s = BN_new()) == NULL)
1747 		{
1748 		PK11err(PK11_F_DSA_SIGN, PK11_R_MALLOC_FAILURE);
1749 		goto ret;
1750 		}
1751 
1752 	if ((r = BN_new()) == NULL)
1753 		{
1754 		PK11err(PK11_F_DSA_SIGN, PK11_R_MALLOC_FAILURE);
1755 		goto ret;
1756 		}
1757 
1758 	if ((dsa_sig = DSA_SIG_new()) == NULL)
1759 		{
1760 		PK11err(PK11_F_DSA_SIGN, PK11_R_MALLOC_FAILURE);
1761 		goto ret;
1762 		}
1763 
1764 	if (BN_bin2bn(sigret, siglen2, r) == NULL ||
1765 	    BN_bin2bn(&sigret[siglen2], siglen2, s) == NULL)
1766 		{
1767 		PK11err(PK11_F_DSA_SIGN, PK11_R_MALLOC_FAILURE);
1768 		goto ret;
1769 		}
1770 
1771 	dsa_sig->r = r;
1772 	dsa_sig->s = s;
1773 
1774 ret:
1775 	if (dsa_sig == NULL)
1776 		{
1777 		if (r != NULL)
1778 			BN_free(r);
1779 		if (s != NULL)
1780 			BN_free(s);
1781 		}
1782 
1783 	pk11_return_session(sp, OP_DSA);
1784 	return (dsa_sig);
1785 	}
1786 
1787 static int
pk11_dsa_do_verify(const unsigned char * dgst,int dlen,DSA_SIG * sig,DSA * dsa)1788 pk11_dsa_do_verify(const unsigned char *dgst, int dlen, DSA_SIG *sig,
1789 	DSA *dsa)
1790 	{
1791 	int i;
1792 	CK_RV rv;
1793 	int retval = 0;
1794 	CK_MECHANISM Mechanism_dsa = {CKM_DSA, NULL, 0};
1795 	CK_MECHANISM *p_mech = &Mechanism_dsa;
1796 	CK_OBJECT_HANDLE h_pub_key;
1797 
1798 	unsigned char sigbuf[DSA_SIGNATURE_LEN];
1799 	unsigned long siglen = DSA_SIGNATURE_LEN;
1800 	unsigned long siglen2 = DSA_SIGNATURE_LEN/2;
1801 
1802 	PK11_SESSION *sp = NULL;
1803 
1804 	if (BN_is_zero(sig->r) || sig->r->neg || BN_ucmp(sig->r, dsa->q) >= 0)
1805 		{
1806 		PK11err(PK11_F_DSA_VERIFY,
1807 			PK11_R_INVALID_DSA_SIGNATURE_R);
1808 		goto ret;
1809 		}
1810 
1811 	if (BN_is_zero(sig->s) || sig->s->neg || BN_ucmp(sig->s, dsa->q) >= 0)
1812 		{
1813 		PK11err(PK11_F_DSA_VERIFY,
1814 			PK11_R_INVALID_DSA_SIGNATURE_S);
1815 		goto ret;
1816 		}
1817 
1818 	i = BN_num_bytes(dsa->q); /* should be 20 */
1819 
1820 	if (dlen > i)
1821 		{
1822 		PK11err(PK11_F_DSA_VERIFY,
1823 			PK11_R_INVALID_SIGNATURE_LENGTH);
1824 		goto ret;
1825 		}
1826 
1827 	if ((sp = pk11_get_session(OP_DSA)) == NULL)
1828 		goto ret;
1829 
1830 	(void) check_new_dsa_key_pub(sp, dsa);
1831 
1832 	h_pub_key = sp->opdata_dsa_pub_key;
1833 	if (h_pub_key == CK_INVALID_HANDLE)
1834 		h_pub_key = sp->opdata_dsa_pub_key =
1835 			pk11_get_public_dsa_key((DSA *)dsa, &sp->opdata_dsa_pub,
1836 			    &sp->opdata_dsa_pub_num, sp->session);
1837 
1838 	if (h_pub_key != CK_INVALID_HANDLE)
1839 		{
1840 		rv = pFuncList->C_VerifyInit(sp->session, p_mech,
1841 			h_pub_key);
1842 
1843 		if (rv != CKR_OK)
1844 			{
1845 			PK11err_add_data(PK11_F_DSA_VERIFY, PK11_R_VERIFYINIT,
1846 			    rv);
1847 			goto ret;
1848 			}
1849 
1850 		/*
1851 		 * The representation of each of the two big numbers could
1852 		 * be shorter than DSA_SIGNATURE_LEN/2 bytes so we need
1853 		 * to act accordingly and shift if necessary.
1854 		 */
1855 		(void) memset(sigbuf, 0, siglen);
1856 		(void) BN_bn2bin(sig->r,
1857 		    sigbuf + siglen2 - BN_num_bytes(sig->r));
1858 		(void) BN_bn2bin(sig->s, &sigbuf[siglen2] + siglen2 -
1859 		    BN_num_bytes(sig->s));
1860 
1861 		rv = pFuncList->C_Verify(sp->session,
1862 			(unsigned char *) dgst, dlen, sigbuf, (CK_ULONG)siglen);
1863 
1864 		if (rv != CKR_OK)
1865 			{
1866 			PK11err_add_data(PK11_F_DSA_VERIFY, PK11_R_VERIFY, rv);
1867 			goto ret;
1868 			}
1869 		}
1870 
1871 	retval = 1;
1872 ret:
1873 
1874 	pk11_return_session(sp, OP_DSA);
1875 	return (retval);
1876 	}
1877 
1878 
1879 /*
1880  * Create a public key object in a session from a given dsa structure.
1881  * The *dsa_pub_num pointer is non-NULL for DSA public keys.
1882  */
pk11_get_public_dsa_key(DSA * dsa,DSA ** key_ptr,BIGNUM ** dsa_pub_num,CK_SESSION_HANDLE session)1883 static CK_OBJECT_HANDLE pk11_get_public_dsa_key(DSA* dsa,
1884     DSA **key_ptr, BIGNUM **dsa_pub_num, CK_SESSION_HANDLE session)
1885 	{
1886 	CK_RV rv;
1887 	CK_OBJECT_CLASS o_key = CKO_PUBLIC_KEY;
1888 	CK_OBJECT_HANDLE h_key = CK_INVALID_HANDLE;
1889 	CK_ULONG found;
1890 	CK_KEY_TYPE k_type = CKK_DSA;
1891 	CK_ULONG ul_key_attr_count = 8;
1892 	CK_BBOOL rollback = FALSE;
1893 	int i;
1894 
1895 	CK_ATTRIBUTE  a_key_template[] =
1896 		{
1897 		{CKA_CLASS, (void *) NULL, sizeof (CK_OBJECT_CLASS)},
1898 		{CKA_KEY_TYPE, (void *) NULL, sizeof (CK_KEY_TYPE)},
1899 		{CKA_TOKEN, &false, sizeof (true)},
1900 		{CKA_VERIFY, &true, sizeof (true)},
1901 		{CKA_PRIME, (void *)NULL, 0},		/* p */
1902 		{CKA_SUBPRIME, (void *)NULL, 0},	/* q */
1903 		{CKA_BASE, (void *)NULL, 0},		/* g */
1904 		{CKA_VALUE, (void *)NULL, 0}		/* pub_key - y */
1905 		};
1906 
1907 	a_key_template[0].pValue = &o_key;
1908 	a_key_template[1].pValue = &k_type;
1909 
1910 	if (init_template_value(dsa->p, &a_key_template[4].pValue,
1911 		&a_key_template[4].ulValueLen) == 0 ||
1912 	    init_template_value(dsa->q, &a_key_template[5].pValue,
1913 		&a_key_template[5].ulValueLen) == 0 ||
1914 	    init_template_value(dsa->g, &a_key_template[6].pValue,
1915 		&a_key_template[6].ulValueLen) == 0 ||
1916 	    init_template_value(dsa->pub_key, &a_key_template[7].pValue,
1917 		&a_key_template[7].ulValueLen) == 0)
1918 		{
1919 		PK11err(PK11_F_GET_PUB_DSA_KEY, PK11_R_MALLOC_FAILURE);
1920 		goto malloc_err;
1921 		}
1922 
1923 	/* see find_lock array definition for more info on object locking */
1924 	LOCK_OBJSTORE(OP_DSA);
1925 	rv = pFuncList->C_FindObjectsInit(session, a_key_template,
1926 		ul_key_attr_count);
1927 
1928 	if (rv != CKR_OK)
1929 		{
1930 		PK11err_add_data(PK11_F_GET_PUB_DSA_KEY, PK11_R_FINDOBJECTSINIT,
1931 		    rv);
1932 		goto err;
1933 		}
1934 
1935 	rv = pFuncList->C_FindObjects(session, &h_key, 1, &found);
1936 
1937 	if (rv != CKR_OK)
1938 		{
1939 		PK11err_add_data(PK11_F_GET_PUB_DSA_KEY,
1940 		    PK11_R_FINDOBJECTS, rv);
1941 		goto err;
1942 		}
1943 
1944 	rv = pFuncList->C_FindObjectsFinal(session);
1945 
1946 	if (rv != CKR_OK)
1947 		{
1948 		PK11err_add_data(PK11_F_GET_PUB_DSA_KEY,
1949 		    PK11_R_FINDOBJECTSFINAL, rv);
1950 		goto err;
1951 		}
1952 
1953 	if (found == 0)
1954 		{
1955 		rv = pFuncList->C_CreateObject(session,
1956 			a_key_template, ul_key_attr_count, &h_key);
1957 		if (rv != CKR_OK)
1958 			{
1959 			PK11err_add_data(PK11_F_GET_PUB_DSA_KEY,
1960 			    PK11_R_CREATEOBJECT, rv);
1961 			goto err;
1962 			}
1963 		}
1964 
1965 	if (dsa_pub_num != NULL)
1966 		if ((*dsa_pub_num = BN_dup(dsa->pub_key)) == NULL)
1967 			{
1968 			PK11err(PK11_F_GET_PUB_DSA_KEY, PK11_R_MALLOC_FAILURE);
1969 			rollback = TRUE;
1970 			goto err;
1971 			}
1972 
1973 	/* LINTED: E_CONSTANT_CONDITION */
1974 	KEY_HANDLE_REFHOLD(h_key, OP_DSA, FALSE, rollback, err);
1975 	if (key_ptr != NULL)
1976 		*key_ptr = dsa;
1977 
1978 err:
1979 	if (rollback)
1980 		{
1981 		/*
1982 		 * We do not care about the return value from C_DestroyObject()
1983 		 * since we are doing rollback.
1984 		 */
1985 		if (found == 0)
1986 			(void) pFuncList->C_DestroyObject(session, h_key);
1987 		h_key = CK_INVALID_HANDLE;
1988 		}
1989 
1990 	UNLOCK_OBJSTORE(OP_DSA);
1991 
1992 malloc_err:
1993 	for (i = 4; i <= 7; i++)
1994 		{
1995 		if (a_key_template[i].pValue != NULL)
1996 			{
1997 			OPENSSL_free(a_key_template[i].pValue);
1998 			a_key_template[i].pValue = NULL;
1999 			}
2000 		}
2001 
2002 	return (h_key);
2003 	}
2004 
2005 /*
2006  * Create a private key object in the session from a given dsa structure
2007  * The *dsa_priv_num pointer is non-NULL for DSA private keys.
2008  */
pk11_get_private_dsa_key(DSA * dsa,DSA ** key_ptr,BIGNUM ** dsa_priv_num,CK_SESSION_HANDLE session)2009 static CK_OBJECT_HANDLE pk11_get_private_dsa_key(DSA* dsa,
2010     DSA **key_ptr, BIGNUM **dsa_priv_num, CK_SESSION_HANDLE session)
2011 	{
2012 	CK_RV rv;
2013 	CK_OBJECT_HANDLE h_key = CK_INVALID_HANDLE;
2014 	CK_OBJECT_CLASS o_key = CKO_PRIVATE_KEY;
2015 	int i;
2016 	CK_ULONG found;
2017 	CK_KEY_TYPE k_type = CKK_DSA;
2018 	CK_ULONG ul_key_attr_count = 9;
2019 	CK_BBOOL rollback = FALSE;
2020 
2021 	/* Both CKA_TOKEN and CKA_SENSITIVE have to be FALSE for session keys */
2022 	CK_ATTRIBUTE  a_key_template[] =
2023 		{
2024 		{CKA_CLASS, (void *) NULL, sizeof (CK_OBJECT_CLASS)},
2025 		{CKA_KEY_TYPE, (void *) NULL, sizeof (CK_KEY_TYPE)},
2026 		{CKA_TOKEN, &false, sizeof (true)},
2027 		{CKA_SENSITIVE, &false, sizeof (true)},
2028 		{CKA_SIGN, &true, sizeof (true)},
2029 		{CKA_PRIME, (void *)NULL, 0},		/* p */
2030 		{CKA_SUBPRIME, (void *)NULL, 0},	/* q */
2031 		{CKA_BASE, (void *)NULL, 0},		/* g */
2032 		{CKA_VALUE, (void *)NULL, 0}		/* priv_key - x */
2033 		};
2034 
2035 	a_key_template[0].pValue = &o_key;
2036 	a_key_template[1].pValue = &k_type;
2037 
2038 	/* Put the private key components into the template */
2039 	if (init_template_value(dsa->p, &a_key_template[5].pValue,
2040 		&a_key_template[5].ulValueLen) == 0 ||
2041 	    init_template_value(dsa->q, &a_key_template[6].pValue,
2042 		&a_key_template[6].ulValueLen) == 0 ||
2043 	    init_template_value(dsa->g, &a_key_template[7].pValue,
2044 		&a_key_template[7].ulValueLen) == 0 ||
2045 	    init_template_value(dsa->priv_key, &a_key_template[8].pValue,
2046 		&a_key_template[8].ulValueLen) == 0)
2047 		{
2048 		PK11err(PK11_F_GET_PRIV_DSA_KEY, PK11_R_MALLOC_FAILURE);
2049 		goto malloc_err;
2050 		}
2051 
2052 	/* see find_lock array definition for more info on object locking */
2053 	LOCK_OBJSTORE(OP_DSA);
2054 	rv = pFuncList->C_FindObjectsInit(session, a_key_template,
2055 		ul_key_attr_count);
2056 
2057 	if (rv != CKR_OK)
2058 		{
2059 		PK11err_add_data(PK11_F_GET_PRIV_DSA_KEY,
2060 		    PK11_R_FINDOBJECTSINIT, rv);
2061 		goto err;
2062 		}
2063 
2064 	rv = pFuncList->C_FindObjects(session, &h_key, 1, &found);
2065 
2066 	if (rv != CKR_OK)
2067 		{
2068 		PK11err_add_data(PK11_F_GET_PRIV_DSA_KEY,
2069 		    PK11_R_FINDOBJECTS, rv);
2070 		goto err;
2071 		}
2072 
2073 	rv = pFuncList->C_FindObjectsFinal(session);
2074 
2075 	if (rv != CKR_OK)
2076 		{
2077 		PK11err_add_data(PK11_F_GET_PRIV_DSA_KEY,
2078 		    PK11_R_FINDOBJECTSFINAL, rv);
2079 		goto err;
2080 		}
2081 
2082 	if (found == 0)
2083 		{
2084 		rv = pFuncList->C_CreateObject(session,
2085 			a_key_template, ul_key_attr_count, &h_key);
2086 		if (rv != CKR_OK)
2087 			{
2088 			PK11err_add_data(PK11_F_GET_PRIV_DSA_KEY,
2089 			    PK11_R_CREATEOBJECT, rv);
2090 			goto err;
2091 			}
2092 		}
2093 
2094 	if (dsa_priv_num != NULL)
2095 		if ((*dsa_priv_num = BN_dup(dsa->priv_key)) == NULL)
2096 			{
2097 			PK11err(PK11_F_GET_PRIV_DSA_KEY, PK11_R_MALLOC_FAILURE);
2098 			rollback = TRUE;
2099 			goto err;
2100 			}
2101 
2102 	/* LINTED: E_CONSTANT_CONDITION */
2103 	KEY_HANDLE_REFHOLD(h_key, OP_DSA, FALSE, rollback, err);
2104 	if (key_ptr != NULL)
2105 		*key_ptr = dsa;
2106 
2107 err:
2108 	if (rollback)
2109 		{
2110 		/*
2111 		 * We do not care about the return value from C_DestroyObject()
2112 		 * since we are doing rollback.
2113 		 */
2114 		if (found == 0)
2115 			(void) pFuncList->C_DestroyObject(session, h_key);
2116 		h_key = CK_INVALID_HANDLE;
2117 		}
2118 
2119 	UNLOCK_OBJSTORE(OP_DSA);
2120 
2121 malloc_err:
2122 	/*
2123 	 * 5 to 8 entries in the key template are key components.
2124 	 * They need to be freed apon exit or error.
2125 	 */
2126 	for (i = 5; i <= 8; i++)
2127 		{
2128 		if (a_key_template[i].pValue != NULL)
2129 			{
2130 			(void) memset(a_key_template[i].pValue, 0,
2131 				a_key_template[i].ulValueLen);
2132 			OPENSSL_free(a_key_template[i].pValue);
2133 			a_key_template[i].pValue = NULL;
2134 			}
2135 		}
2136 
2137 	return (h_key);
2138 	}
2139 
2140 /*
2141  * Check for cache miss and clean the object pointer and handle
2142  * in such case. Return 1 for cache hit, 0 for cache miss.
2143  */
check_new_dsa_key_pub(PK11_SESSION * sp,DSA * dsa)2144 static int check_new_dsa_key_pub(PK11_SESSION *sp, DSA *dsa)
2145 	{
2146 	/*
2147 	 * Provide protection against DSA structure reuse by making the
2148 	 * check for cache hit stronger. Only public key component of DSA
2149 	 * key matters here so it is sufficient to compare it with value
2150 	 * cached in PK11_SESSION structure.
2151 	 */
2152 	if ((sp->opdata_dsa_pub != dsa) ||
2153 	    (BN_cmp(sp->opdata_dsa_pub_num, dsa->pub_key) != 0))
2154 		{
2155 		/*
2156 		 * We do not check the return value because even in case of
2157 		 * failure the sp structure will have both key pointer
2158 		 * and object handle cleaned and pk11_destroy_object()
2159 		 * reports the failure to the OpenSSL error message buffer.
2160 		 */
2161 		(void) pk11_destroy_dsa_object_pub(sp, TRUE);
2162 		return (0);
2163 		}
2164 	return (1);
2165 	}
2166 
2167 /*
2168  * Check for cache miss and clean the object pointer and handle
2169  * in such case. Return 1 for cache hit, 0 for cache miss.
2170  */
check_new_dsa_key_priv(PK11_SESSION * sp,DSA * dsa)2171 static int check_new_dsa_key_priv(PK11_SESSION *sp, DSA *dsa)
2172 	{
2173 	/*
2174 	 * Provide protection against DSA structure reuse by making the
2175 	 * check for cache hit stronger. Only private key component of DSA
2176 	 * key matters here so it is sufficient to compare it with value
2177 	 * cached in PK11_SESSION structure.
2178 	 */
2179 	if ((sp->opdata_dsa_priv != dsa) ||
2180 	    (BN_cmp(sp->opdata_dsa_priv_num, dsa->priv_key) != 0))
2181 		{
2182 		/*
2183 		 * We do not check the return value because even in case of
2184 		 * failure the sp structure will have both key pointer
2185 		 * and object handle cleaned and pk11_destroy_object()
2186 		 * reports the failure to the OpenSSL error message buffer.
2187 		 */
2188 		(void) pk11_destroy_dsa_object_priv(sp, TRUE);
2189 		return (0);
2190 		}
2191 	return (1);
2192 	}
2193 #endif
2194 
2195 
2196 #ifndef OPENSSL_NO_DH
2197 /* The DH function implementation */
2198 /* ARGSUSED */
pk11_DH_init(DH * dh)2199 static int pk11_DH_init(DH *dh)
2200 	{
2201 	return (1);
2202 	}
2203 
2204 /* ARGSUSED */
pk11_DH_finish(DH * dh)2205 static int pk11_DH_finish(DH *dh)
2206 	{
2207 	return (1);
2208 	}
2209 
2210 /*
2211  * Generate DH key-pair.
2212  *
2213  * Warning: Unlike OpenSSL's DH_generate_key(3) we ignore dh->priv_key
2214  * and override it even if it is set. OpenSSL does not touch dh->priv_key
2215  * if set and just computes dh->pub_key. It looks like PKCS#11 standard
2216  * is not capable of providing this functionality. This could be a problem
2217  * for applications relying on OpenSSL's semantics.
2218  */
pk11_DH_generate_key(DH * dh)2219 static int pk11_DH_generate_key(DH *dh)
2220 	{
2221 	CK_ULONG i;
2222 	CK_RV rv, rv1;
2223 	int reuse_mem_len = 0, ret = 0;
2224 	PK11_SESSION *sp = NULL;
2225 	CK_BYTE_PTR reuse_mem;
2226 
2227 	CK_MECHANISM mechanism = {CKM_DH_PKCS_KEY_PAIR_GEN, NULL_PTR, 0};
2228 	CK_OBJECT_HANDLE h_pub_key = CK_INVALID_HANDLE;
2229 	CK_OBJECT_HANDLE h_priv_key = CK_INVALID_HANDLE;
2230 
2231 	CK_ULONG ul_pub_key_attr_count = 3;
2232 	CK_ATTRIBUTE pub_key_template[] =
2233 		{
2234 		{CKA_PRIVATE, &false, sizeof (false)},
2235 		{CKA_PRIME, (void *)NULL, 0},
2236 		{CKA_BASE, (void *)NULL, 0}
2237 		};
2238 
2239 	CK_ULONG ul_priv_key_attr_count = 3;
2240 	CK_ATTRIBUTE priv_key_template[] =
2241 		{
2242 		{CKA_PRIVATE, &false, sizeof (false)},
2243 		{CKA_SENSITIVE, &false, sizeof (false)},
2244 		{CKA_DERIVE, &true, sizeof (true)}
2245 		};
2246 
2247 	CK_ULONG pub_key_attr_result_count = 1;
2248 	CK_ATTRIBUTE pub_key_result[] =
2249 		{
2250 		{CKA_VALUE, (void *)NULL, 0}
2251 		};
2252 
2253 	CK_ULONG priv_key_attr_result_count = 1;
2254 	CK_ATTRIBUTE priv_key_result[] =
2255 		{
2256 		{CKA_VALUE, (void *)NULL, 0}
2257 		};
2258 
2259 	pub_key_template[1].ulValueLen = BN_num_bytes(dh->p);
2260 	if (pub_key_template[1].ulValueLen > 0)
2261 		{
2262 		/*
2263 		 * We must not increase ulValueLen by DH_BUF_RESERVE since that
2264 		 * could cause the same rounding problem. See definition of
2265 		 * DH_BUF_RESERVE above.
2266 		 */
2267 		pub_key_template[1].pValue =
2268 			OPENSSL_malloc(pub_key_template[1].ulValueLen +
2269 			DH_BUF_RESERVE);
2270 		if (pub_key_template[1].pValue == NULL)
2271 			{
2272 			PK11err(PK11_F_DH_GEN_KEY, PK11_R_MALLOC_FAILURE);
2273 			goto err;
2274 			}
2275 
2276 		i = BN_bn2bin(dh->p, pub_key_template[1].pValue);
2277 		}
2278 	else
2279 		goto err;
2280 
2281 	pub_key_template[2].ulValueLen = BN_num_bytes(dh->g);
2282 	if (pub_key_template[2].ulValueLen > 0)
2283 		{
2284 		pub_key_template[2].pValue =
2285 			OPENSSL_malloc(pub_key_template[2].ulValueLen +
2286 			DH_BUF_RESERVE);
2287 		if (pub_key_template[2].pValue == NULL)
2288 			{
2289 			PK11err(PK11_F_DH_GEN_KEY, PK11_R_MALLOC_FAILURE);
2290 			goto err;
2291 			}
2292 
2293 		i = BN_bn2bin(dh->g, pub_key_template[2].pValue);
2294 		}
2295 	else
2296 		goto err;
2297 
2298 	/*
2299 	 * Note: we are only using PK11_SESSION structure for getting
2300 	 *	 a session handle. The objects created in this function are
2301 	 *	 destroyed before return and thus not cached.
2302 	 */
2303 	if ((sp = pk11_get_session(OP_DH)) == NULL)
2304 		goto err;
2305 
2306 	rv = pFuncList->C_GenerateKeyPair(sp->session,
2307 	    &mechanism,
2308 	    pub_key_template,
2309 	    ul_pub_key_attr_count,
2310 	    priv_key_template,
2311 	    ul_priv_key_attr_count,
2312 	    &h_pub_key,
2313 	    &h_priv_key);
2314 	if (rv != CKR_OK)
2315 		{
2316 		PK11err_add_data(PK11_F_DH_GEN_KEY, PK11_R_GEN_KEY, rv);
2317 		goto err;
2318 		}
2319 
2320 	/*
2321 	 * Reuse the larger memory allocated. We know the larger memory
2322 	 * should be sufficient for reuse.
2323 	 */
2324 	if (pub_key_template[1].ulValueLen > pub_key_template[2].ulValueLen)
2325 		{
2326 		reuse_mem = pub_key_template[1].pValue;
2327 		reuse_mem_len = pub_key_template[1].ulValueLen + DH_BUF_RESERVE;
2328 		}
2329 	else
2330 		{
2331 		reuse_mem = pub_key_template[2].pValue;
2332 		reuse_mem_len = pub_key_template[2].ulValueLen + DH_BUF_RESERVE;
2333 		}
2334 
2335 	rv = pFuncList->C_GetAttributeValue(sp->session, h_pub_key,
2336 		pub_key_result, pub_key_attr_result_count);
2337 	rv1 = pFuncList->C_GetAttributeValue(sp->session, h_priv_key,
2338 		priv_key_result, priv_key_attr_result_count);
2339 
2340 	if (rv != CKR_OK || rv1 != CKR_OK)
2341 		{
2342 		rv = (rv != CKR_OK) ? rv : rv1;
2343 		PK11err_add_data(PK11_F_DH_GEN_KEY,
2344 		    PK11_R_GETATTRIBUTVALUE, rv);
2345 		goto err;
2346 		}
2347 
2348 	if (((CK_LONG) pub_key_result[0].ulValueLen) <= 0 ||
2349 		((CK_LONG) priv_key_result[0].ulValueLen) <= 0)
2350 		{
2351 		PK11err(PK11_F_DH_GEN_KEY, PK11_R_GETATTRIBUTVALUE);
2352 		goto err;
2353 		}
2354 
2355 	/* Reuse the memory allocated */
2356 	pub_key_result[0].pValue = reuse_mem;
2357 	pub_key_result[0].ulValueLen = reuse_mem_len;
2358 
2359 	rv = pFuncList->C_GetAttributeValue(sp->session, h_pub_key,
2360 		pub_key_result, pub_key_attr_result_count);
2361 
2362 	if (rv != CKR_OK)
2363 		{
2364 		PK11err_add_data(PK11_F_DH_GEN_KEY,
2365 		    PK11_R_GETATTRIBUTVALUE, rv);
2366 		goto err;
2367 		}
2368 
2369 	if (pub_key_result[0].type == CKA_VALUE)
2370 		{
2371 		if (dh->pub_key == NULL)
2372 			if ((dh->pub_key = BN_new()) == NULL)
2373 				{
2374 				PK11err(PK11_F_DH_GEN_KEY,
2375 					PK11_R_MALLOC_FAILURE);
2376 				goto err;
2377 				}
2378 		dh->pub_key = BN_bin2bn(pub_key_result[0].pValue,
2379 			pub_key_result[0].ulValueLen, dh->pub_key);
2380 		if (dh->pub_key == NULL)
2381 			{
2382 			PK11err(PK11_F_DH_GEN_KEY, PK11_R_MALLOC_FAILURE);
2383 			goto err;
2384 			}
2385 		}
2386 
2387 	/* Reuse the memory allocated */
2388 	priv_key_result[0].pValue = reuse_mem;
2389 	priv_key_result[0].ulValueLen = reuse_mem_len;
2390 
2391 	rv = pFuncList->C_GetAttributeValue(sp->session, h_priv_key,
2392 		priv_key_result, priv_key_attr_result_count);
2393 
2394 	if (rv != CKR_OK)
2395 		{
2396 		PK11err_add_data(PK11_F_DH_GEN_KEY,
2397 		    PK11_R_GETATTRIBUTVALUE, rv);
2398 		goto err;
2399 		}
2400 
2401 	if (priv_key_result[0].type == CKA_VALUE)
2402 		{
2403 		if (dh->priv_key == NULL)
2404 			if ((dh->priv_key = BN_new()) == NULL)
2405 				{
2406 				PK11err(PK11_F_DH_GEN_KEY,
2407 					PK11_R_MALLOC_FAILURE);
2408 				goto err;
2409 				}
2410 		dh->priv_key = BN_bin2bn(priv_key_result[0].pValue,
2411 			priv_key_result[0].ulValueLen, dh->priv_key);
2412 		if (dh->priv_key == NULL)
2413 			{
2414 			PK11err(PK11_F_DH_GEN_KEY, PK11_R_MALLOC_FAILURE);
2415 			goto err;
2416 			}
2417 		}
2418 
2419 	ret = 1;
2420 
2421 err:
2422 
2423 	if (h_pub_key != CK_INVALID_HANDLE)
2424 		{
2425 		rv = pFuncList->C_DestroyObject(sp->session, h_pub_key);
2426 		if (rv != CKR_OK)
2427 			{
2428 			PK11err_add_data(PK11_F_DH_GEN_KEY,
2429 			    PK11_R_DESTROYOBJECT, rv);
2430 			}
2431 		}
2432 
2433 	if (h_priv_key != CK_INVALID_HANDLE)
2434 		{
2435 		rv = pFuncList->C_DestroyObject(sp->session, h_priv_key);
2436 		if (rv != CKR_OK)
2437 			{
2438 			PK11err_add_data(PK11_F_DH_GEN_KEY,
2439 			    PK11_R_DESTROYOBJECT, rv);
2440 			}
2441 		}
2442 
2443 	for (i = 1; i <= 2; i++)
2444 		{
2445 		if (pub_key_template[i].pValue != NULL)
2446 			{
2447 			OPENSSL_free(pub_key_template[i].pValue);
2448 			pub_key_template[i].pValue = NULL;
2449 			}
2450 		}
2451 
2452 	pk11_return_session(sp, OP_DH);
2453 	return (ret);
2454 	}
2455 
pk11_DH_compute_key(unsigned char * key,const BIGNUM * pub_key,DH * dh)2456 static int pk11_DH_compute_key(unsigned char *key, const BIGNUM *pub_key,
2457 	DH *dh)
2458 	{
2459 	int i;
2460 	CK_MECHANISM mechanism = {CKM_DH_PKCS_DERIVE, NULL_PTR, 0};
2461 	CK_OBJECT_CLASS key_class = CKO_SECRET_KEY;
2462 	CK_KEY_TYPE key_type = CKK_GENERIC_SECRET;
2463 	CK_OBJECT_HANDLE h_derived_key = CK_INVALID_HANDLE;
2464 	CK_OBJECT_HANDLE h_key = CK_INVALID_HANDLE;
2465 
2466 	CK_ULONG ul_priv_key_attr_count = 2;
2467 	CK_ATTRIBUTE priv_key_template[] =
2468 		{
2469 		{CKA_CLASS, (void*) NULL, sizeof (key_class)},
2470 		{CKA_KEY_TYPE, (void*) NULL, sizeof (key_type)},
2471 		};
2472 
2473 	CK_ULONG priv_key_attr_result_count = 1;
2474 	CK_ATTRIBUTE priv_key_result[] =
2475 		{
2476 		{CKA_VALUE, (void *)NULL, 0}
2477 		};
2478 
2479 	CK_RV rv;
2480 	int ret = -1;
2481 	PK11_SESSION *sp = NULL;
2482 
2483 	if (dh->priv_key == NULL)
2484 		goto err;
2485 
2486 	priv_key_template[0].pValue = &key_class;
2487 	priv_key_template[1].pValue = &key_type;
2488 
2489 	if ((sp = pk11_get_session(OP_DH)) == NULL)
2490 		goto err;
2491 
2492 	mechanism.ulParameterLen = BN_num_bytes(pub_key);
2493 	mechanism.pParameter = OPENSSL_malloc(mechanism.ulParameterLen);
2494 	if (mechanism.pParameter == NULL)
2495 		{
2496 		PK11err(PK11_F_DH_COMP_KEY, PK11_R_MALLOC_FAILURE);
2497 		goto err;
2498 		}
2499 	(void) BN_bn2bin(pub_key, mechanism.pParameter);
2500 
2501 	(void) check_new_dh_key(sp, dh);
2502 
2503 	h_key = sp->opdata_dh_key;
2504 	if (h_key == CK_INVALID_HANDLE)
2505 		h_key = sp->opdata_dh_key =
2506 			pk11_get_dh_key((DH*) dh, &sp->opdata_dh,
2507 			    &sp->opdata_dh_priv_num, sp->session);
2508 
2509 	if (h_key == CK_INVALID_HANDLE)
2510 		{
2511 		PK11err(PK11_F_DH_COMP_KEY, PK11_R_CREATEOBJECT);
2512 		goto err;
2513 		}
2514 
2515 	rv = pFuncList->C_DeriveKey(sp->session,
2516 	    &mechanism,
2517 	    h_key,
2518 	    priv_key_template,
2519 	    ul_priv_key_attr_count,
2520 	    &h_derived_key);
2521 	if (rv != CKR_OK)
2522 		{
2523 		PK11err_add_data(PK11_F_DH_COMP_KEY, PK11_R_DERIVEKEY, rv);
2524 		goto err;
2525 		}
2526 
2527 	rv = pFuncList->C_GetAttributeValue(sp->session, h_derived_key,
2528 	    priv_key_result, priv_key_attr_result_count);
2529 
2530 	if (rv != CKR_OK)
2531 		{
2532 		PK11err_add_data(PK11_F_DH_COMP_KEY, PK11_R_GETATTRIBUTVALUE,
2533 		    rv);
2534 		goto err;
2535 		}
2536 
2537 	if (((CK_LONG) priv_key_result[0].ulValueLen) <= 0)
2538 		{
2539 		PK11err(PK11_F_DH_COMP_KEY, PK11_R_GETATTRIBUTVALUE);
2540 		goto err;
2541 		}
2542 	priv_key_result[0].pValue =
2543 		OPENSSL_malloc(priv_key_result[0].ulValueLen);
2544 	if (!priv_key_result[0].pValue)
2545 		{
2546 		PK11err(PK11_F_DH_COMP_KEY, PK11_R_MALLOC_FAILURE);
2547 		goto err;
2548 		}
2549 
2550 	rv = pFuncList->C_GetAttributeValue(sp->session, h_derived_key,
2551 		priv_key_result, priv_key_attr_result_count);
2552 
2553 	if (rv != CKR_OK)
2554 		{
2555 		PK11err_add_data(PK11_F_DH_COMP_KEY, PK11_R_GETATTRIBUTVALUE,
2556 		    rv);
2557 		goto err;
2558 		}
2559 
2560 	/*
2561 	 * OpenSSL allocates the output buffer 'key' which is the same
2562 	 * length of the public key. It is long enough for the derived key
2563 	 */
2564 	if (priv_key_result[0].type == CKA_VALUE)
2565 		{
2566 		/*
2567 		 * CKM_DH_PKCS_DERIVE mechanism is not supposed to strip
2568 		 * leading zeros from a computed shared secret. However,
2569 		 * OpenSSL always did it so we must do the same here. The
2570 		 * vagueness of the spec regarding leading zero bytes was
2571 		 * finally cleared with TLS 1.1 (RFC 4346) saying that leading
2572 		 * zeros are stripped before the computed data is used as the
2573 		 * pre-master secret.
2574 		 */
2575 		for (i = 0; i < priv_key_result[0].ulValueLen; ++i)
2576 			{
2577 			if (((char *)priv_key_result[0].pValue)[i] != 0)
2578 				break;
2579 			}
2580 
2581 		(void) memcpy(key, ((char *)priv_key_result[0].pValue) + i,
2582 			priv_key_result[0].ulValueLen - i);
2583 		ret = priv_key_result[0].ulValueLen - i;
2584 		}
2585 
2586 err:
2587 
2588 	if (h_derived_key != CK_INVALID_HANDLE)
2589 		{
2590 		rv = pFuncList->C_DestroyObject(sp->session, h_derived_key);
2591 		if (rv != CKR_OK)
2592 			{
2593 			PK11err_add_data(PK11_F_DH_COMP_KEY,
2594 			    PK11_R_DESTROYOBJECT, rv);
2595 			}
2596 		}
2597 	if (priv_key_result[0].pValue)
2598 		{
2599 		OPENSSL_free(priv_key_result[0].pValue);
2600 		priv_key_result[0].pValue = NULL;
2601 		}
2602 
2603 	if (mechanism.pParameter)
2604 		{
2605 		OPENSSL_free(mechanism.pParameter);
2606 		mechanism.pParameter = NULL;
2607 		}
2608 
2609 	pk11_return_session(sp, OP_DH);
2610 	return (ret);
2611 	}
2612 
2613 
pk11_get_dh_key(DH * dh,DH ** key_ptr,BIGNUM ** dh_priv_num,CK_SESSION_HANDLE session)2614 static CK_OBJECT_HANDLE pk11_get_dh_key(DH* dh,
2615 	DH **key_ptr, BIGNUM **dh_priv_num, CK_SESSION_HANDLE session)
2616 	{
2617 	CK_RV rv;
2618 	CK_OBJECT_HANDLE h_key = CK_INVALID_HANDLE;
2619 	CK_OBJECT_CLASS class = CKO_PRIVATE_KEY;
2620 	CK_KEY_TYPE key_type = CKK_DH;
2621 	CK_ULONG found;
2622 	CK_BBOOL rollback = FALSE;
2623 	int i;
2624 
2625 	CK_ULONG ul_key_attr_count = 7;
2626 	CK_ATTRIBUTE key_template[] =
2627 		{
2628 		{CKA_CLASS, (void*) NULL, sizeof (class)},
2629 		{CKA_KEY_TYPE, (void*) NULL, sizeof (key_type)},
2630 		{CKA_DERIVE, &true, sizeof (true)},
2631 		{CKA_PRIVATE, &false, sizeof (false)},
2632 		{CKA_PRIME, (void *) NULL, 0},
2633 		{CKA_BASE, (void *) NULL, 0},
2634 		{CKA_VALUE, (void *) NULL, 0},
2635 		};
2636 
2637 	key_template[0].pValue = &class;
2638 	key_template[1].pValue = &key_type;
2639 
2640 	key_template[4].ulValueLen = BN_num_bytes(dh->p);
2641 	key_template[4].pValue = (CK_VOID_PTR)OPENSSL_malloc(
2642 		(size_t)key_template[4].ulValueLen);
2643 	if (key_template[4].pValue == NULL)
2644 		{
2645 		PK11err(PK11_F_GET_DH_KEY, PK11_R_MALLOC_FAILURE);
2646 		goto malloc_err;
2647 		}
2648 
2649 	(void) BN_bn2bin(dh->p, key_template[4].pValue);
2650 
2651 	key_template[5].ulValueLen = BN_num_bytes(dh->g);
2652 	key_template[5].pValue = (CK_VOID_PTR)OPENSSL_malloc(
2653 		(size_t)key_template[5].ulValueLen);
2654 	if (key_template[5].pValue == NULL)
2655 		{
2656 		PK11err(PK11_F_GET_DH_KEY, PK11_R_MALLOC_FAILURE);
2657 		goto malloc_err;
2658 		}
2659 
2660 	(void) BN_bn2bin(dh->g, key_template[5].pValue);
2661 
2662 	key_template[6].ulValueLen = BN_num_bytes(dh->priv_key);
2663 	key_template[6].pValue = (CK_VOID_PTR)OPENSSL_malloc(
2664 		(size_t)key_template[6].ulValueLen);
2665 	if (key_template[6].pValue == NULL)
2666 		{
2667 		PK11err(PK11_F_GET_DH_KEY, PK11_R_MALLOC_FAILURE);
2668 		goto malloc_err;
2669 		}
2670 
2671 	(void) BN_bn2bin(dh->priv_key, key_template[6].pValue);
2672 
2673 	/* see find_lock array definition for more info on object locking */
2674 	LOCK_OBJSTORE(OP_DH);
2675 	rv = pFuncList->C_FindObjectsInit(session, key_template,
2676 		ul_key_attr_count);
2677 
2678 	if (rv != CKR_OK)
2679 		{
2680 		PK11err_add_data(PK11_F_GET_DH_KEY, PK11_R_FINDOBJECTSINIT, rv);
2681 		goto err;
2682 		}
2683 
2684 	rv = pFuncList->C_FindObjects(session, &h_key, 1, &found);
2685 
2686 	if (rv != CKR_OK)
2687 		{
2688 		PK11err_add_data(PK11_F_GET_DH_KEY, PK11_R_FINDOBJECTS, rv);
2689 		goto err;
2690 		}
2691 
2692 	rv = pFuncList->C_FindObjectsFinal(session);
2693 
2694 	if (rv != CKR_OK)
2695 		{
2696 		PK11err_add_data(PK11_F_GET_DH_KEY, PK11_R_FINDOBJECTSFINAL,
2697 		    rv);
2698 		goto err;
2699 		}
2700 
2701 	if (found == 0)
2702 		{
2703 		rv = pFuncList->C_CreateObject(session,
2704 			key_template, ul_key_attr_count, &h_key);
2705 		if (rv != CKR_OK)
2706 			{
2707 			PK11err_add_data(PK11_F_GET_DH_KEY, PK11_R_CREATEOBJECT,
2708 			    rv);
2709 			goto err;
2710 			}
2711 		}
2712 
2713 	if (dh_priv_num != NULL)
2714 		if ((*dh_priv_num = BN_dup(dh->priv_key)) == NULL)
2715 			{
2716 			PK11err(PK11_F_GET_DH_KEY, PK11_R_MALLOC_FAILURE);
2717 			rollback = TRUE;
2718 			goto err;
2719 			}
2720 
2721 	/* LINTED: E_CONSTANT_CONDITION */
2722 	KEY_HANDLE_REFHOLD(h_key, OP_DH, FALSE, rollback, err);
2723 	if (key_ptr != NULL)
2724 		*key_ptr = dh;
2725 
2726 err:
2727 	if (rollback)
2728 		{
2729 		/*
2730 		 * We do not care about the return value from C_DestroyObject()
2731 		 * since we are doing rollback.
2732 		 */
2733 		if (found == 0)
2734 			(void) pFuncList->C_DestroyObject(session, h_key);
2735 		h_key = CK_INVALID_HANDLE;
2736 		}
2737 
2738 	UNLOCK_OBJSTORE(OP_DH);
2739 
2740 malloc_err:
2741 	for (i = 4; i <= 6; i++)
2742 		{
2743 		if (key_template[i].pValue != NULL)
2744 			{
2745 			OPENSSL_free(key_template[i].pValue);
2746 			key_template[i].pValue = NULL;
2747 			}
2748 		}
2749 
2750 	return (h_key);
2751 	}
2752 
2753 /*
2754  * Check for cache miss and clean the object pointer and handle
2755  * in such case. Return 1 for cache hit, 0 for cache miss.
2756  *
2757  * Note: we rely on pk11_destroy_dh_key_objects() to set sp->opdata_dh
2758  *       to CK_INVALID_HANDLE even when it fails to destroy the object.
2759  */
check_new_dh_key(PK11_SESSION * sp,DH * dh)2760 static int check_new_dh_key(PK11_SESSION *sp, DH *dh)
2761 	{
2762 	/*
2763 	 * Provide protection against DH structure reuse by making the
2764 	 * check for cache hit stronger. Private key component of DH key
2765 	 * is unique so it is sufficient to compare it with value cached
2766 	 * in PK11_SESSION structure.
2767 	 */
2768 	if ((sp->opdata_dh != dh) ||
2769 	    (BN_cmp(sp->opdata_dh_priv_num, dh->priv_key) != 0))
2770 		{
2771 		/*
2772 		 * We do not check the return value because even in case of
2773 		 * failure the sp structure will have both key pointer
2774 		 * and object handle cleaned and pk11_destroy_object()
2775 		 * reports the failure to the OpenSSL error message buffer.
2776 		 */
2777 		(void) pk11_destroy_dh_object(sp, TRUE);
2778 		return (0);
2779 		}
2780 	return (1);
2781 	}
2782 #endif
2783 
2784 /*
2785  * Local function to simplify key template population
2786  * Return 0 -- error, 1 -- no error
2787  */
init_template_value(BIGNUM * bn,CK_VOID_PTR * p_value,CK_ULONG * ul_value_len)2788 static int init_template_value(BIGNUM *bn, CK_VOID_PTR *p_value,
2789 	CK_ULONG *ul_value_len)
2790 	{
2791 	CK_ULONG len = BN_num_bytes(bn);
2792 	if (len == 0)
2793 		return (1);
2794 
2795 	*ul_value_len = len;
2796 	*p_value = (CK_VOID_PTR)OPENSSL_malloc((size_t)*ul_value_len);
2797 	if (*p_value == NULL)
2798 		return (0);
2799 
2800 	(void) BN_bn2bin(bn, *p_value);
2801 
2802 	return (1);
2803 	}
2804 
2805 #endif	/* OPENSSL_NO_HW_PK11 */
2806 #endif	/* OPENSSL_NO_HW */
2807