xref: /netbsd-src/crypto/external/bsd/netpgp/dist/src/lib/openssl_crypto.c (revision 600b302b23a1a61fbb5a5cbbdb1b46006bfeccbd)
1 /*-
2  * Copyright (c) 2009 The NetBSD Foundation, Inc.
3  * All rights reserved.
4  *
5  * This code is derived from software contributed to The NetBSD Foundation
6  * by Alistair Crooks (agc@NetBSD.org)
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27  * POSSIBILITY OF SUCH DAMAGE.
28  */
29 /*
30  * Copyright (c) 2005-2008 Nominet UK (www.nic.uk)
31  * All rights reserved.
32  * Contributors: Ben Laurie, Rachel Willmer. The Contributors have asserted
33  * their moral rights under the UK Copyright Design and Patents Act 1988 to
34  * be recorded as the authors of this copyright work.
35  *
36  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
37  * use this file except in compliance with the License.
38  *
39  * You may obtain a copy of the License at
40  *     http://www.apache.org/licenses/LICENSE-2.0
41  *
42  * Unless required by applicable law or agreed to in writing, software
43  * distributed under the License is distributed on an "AS IS" BASIS,
44  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
45  *
46  * See the License for the specific language governing permissions and
47  * limitations under the License.
48  */
49 
50 /** \file
51  */
52 #include "config.h"
53 
54 #ifdef HAVE_SYS_CDEFS_H
55 #include <sys/cdefs.h>
56 #endif
57 
58 #if defined(__NetBSD__)
59 __COPYRIGHT("@(#) Copyright (c) 2009 The NetBSD Foundation, Inc. All rights reserved.");
60 __RCSID("$NetBSD: openssl_crypto.c,v 1.19 2010/02/06 02:24:33 agc Exp $");
61 #endif
62 
63 #ifdef HAVE_OPENSSL_DSA_H
64 #include <openssl/dsa.h>
65 #endif
66 
67 #ifdef HAVE_OPENSSL_RSA_H
68 #include <openssl/rsa.h>
69 #endif
70 
71 #ifdef HAVE_OPENSSL_ERR_H
72 #include <openssl/err.h>
73 #endif
74 
75 #include <openssl/pem.h>
76 #include <openssl/evp.h>
77 
78 #include <stdlib.h>
79 #include <string.h>
80 /* Hash size for secret key check */
81 
82 #include "crypto.h"
83 #include "keyring.h"
84 #include "readerwriter.h"
85 #include "netpgpdefs.h"
86 #include "netpgpdigest.h"
87 #include "packet.h"
88 
89 
90 static void
91 test_seckey(const __ops_seckey_t *seckey)
92 {
93 	RSA            *test = RSA_new();
94 
95 	test->n = BN_dup(seckey->pubkey.key.rsa.n);
96 	test->e = BN_dup(seckey->pubkey.key.rsa.e);
97 
98 	test->d = BN_dup(seckey->key.rsa.d);
99 	test->p = BN_dup(seckey->key.rsa.p);
100 	test->q = BN_dup(seckey->key.rsa.q);
101 
102 	if (RSA_check_key(test) != 1) {
103 		(void) fprintf(stderr,
104 			"test_seckey: RSA_check_key failed\n");
105 	}
106 	RSA_free(test);
107 }
108 
109 static int
110 md5_init(__ops_hash_t *hash)
111 {
112 	if (hash->data) {
113 		(void) fprintf(stderr, "md5_init: hash data non-null\n");
114 	}
115 	if ((hash->data = calloc(1, sizeof(MD5_CTX))) == NULL) {
116 		(void) fprintf(stderr, "md5_init: bad alloc\n");
117 		return 0;
118 	}
119 	MD5_Init(hash->data);
120 	return 1;
121 }
122 
123 static void
124 md5_add(__ops_hash_t *hash, const unsigned char *data, unsigned length)
125 {
126 	MD5_Update(hash->data, data, length);
127 }
128 
129 static unsigned
130 md5_finish(__ops_hash_t *hash, unsigned char *out)
131 {
132 	MD5_Final(out, hash->data);
133 	free(hash->data);
134 	hash->data = NULL;
135 	return 16;
136 }
137 
138 static const __ops_hash_t md5 = {
139 	OPS_HASH_MD5,
140 	MD5_DIGEST_LENGTH,
141 	"MD5",
142 	md5_init,
143 	md5_add,
144 	md5_finish,
145 	NULL
146 };
147 
148 /**
149    \ingroup Core_Crypto
150    \brief Initialise to MD5
151    \param hash Hash to initialise
152 */
153 void
154 __ops_hash_md5(__ops_hash_t *hash)
155 {
156 	*hash = md5;
157 }
158 
159 static int
160 sha1_init(__ops_hash_t *hash)
161 {
162 	if (__ops_get_debug_level(__FILE__)) {
163 		fprintf(stderr, "***\n***\nsha1_init\n***\n");
164 	}
165 	if (hash->data) {
166 		(void) fprintf(stderr, "sha1_init: hash data non-null\n");
167 	}
168 	if ((hash->data = calloc(1, sizeof(SHA_CTX))) == NULL) {
169 		(void) fprintf(stderr, "sha1_init: bad alloc\n");
170 		return 0;
171 	}
172 	SHA1_Init(hash->data);
173 	return 1;
174 }
175 
176 static void
177 sha1_add(__ops_hash_t *hash, const unsigned char *data, unsigned length)
178 {
179 	if (__ops_get_debug_level(__FILE__)) {
180 		unsigned int    i;
181 
182 		(void) fprintf(stderr, "adding %u to hash:\n ", length);
183 		for (i = 0; i < length; i++) {
184 			(void) fprintf(stderr, "0x%02x ", data[i]);
185 			if (!((i + 1) % 16)) {
186 				(void) fprintf(stderr, "\n");
187 			} else if (!((i + 1) % 8)) {
188 				(void) fprintf(stderr, "  ");
189 			}
190 		}
191 		(void) fprintf(stderr, "\n");
192 	}
193 	SHA1_Update(hash->data, data, length);
194 }
195 
196 static unsigned
197 sha1_finish(__ops_hash_t *hash, unsigned char *out)
198 {
199 	SHA1_Final(out, hash->data);
200 	if (__ops_get_debug_level(__FILE__)) {
201 		unsigned        i;
202 
203 		(void) fprintf(stderr, "***\n***\nsha1_finish\n***\n");
204 		for (i = 0; i < OPS_SHA1_HASH_SIZE; i++)
205 			(void) fprintf(stderr, "0x%02x ", out[i]);
206 		(void) fprintf(stderr, "\n");
207 	}
208 	free(hash->data);
209 	hash->data = NULL;
210 	return OPS_SHA1_HASH_SIZE;
211 }
212 
213 static const __ops_hash_t sha1 = {
214 	OPS_HASH_SHA1,
215 	OPS_SHA1_HASH_SIZE,
216 	"SHA1",
217 	sha1_init,
218 	sha1_add,
219 	sha1_finish,
220 	NULL
221 };
222 
223 /**
224    \ingroup Core_Crypto
225    \brief Initialise to SHA1
226    \param hash Hash to initialise
227 */
228 void
229 __ops_hash_sha1(__ops_hash_t *hash)
230 {
231 	*hash = sha1;
232 }
233 
234 static int
235 sha256_init(__ops_hash_t *hash)
236 {
237 	if (__ops_get_debug_level(__FILE__)) {
238 		fprintf(stderr, "***\n***\nsha256_init\n***\n");
239 	}
240 	if (hash->data) {
241 		(void) fprintf(stderr, "sha256_init: hash data non-null\n");
242 	}
243 	if ((hash->data = calloc(1, sizeof(SHA256_CTX))) == NULL) {
244 		(void) fprintf(stderr, "sha256_init: bad alloc\n");
245 		return 0;
246 	}
247 	SHA256_Init(hash->data);
248 	return 1;
249 }
250 
251 static void
252 sha256_add(__ops_hash_t *hash, const unsigned char *data, unsigned length)
253 {
254 	if (__ops_get_debug_level(__FILE__)) {
255 		unsigned int    i;
256 
257 		(void) fprintf(stderr, "adding %d to hash:\n ", length);
258 		for (i = 0; i < length; i++) {
259 			(void) fprintf(stderr, "0x%02x ", data[i]);
260 			if (!((i + 1) % 16))
261 				(void) fprintf(stderr, "\n");
262 			else if (!((i + 1) % 8))
263 				(void) fprintf(stderr, "  ");
264 		}
265 		(void) fprintf(stderr, "\n");
266 	}
267 	SHA256_Update(hash->data, data, length);
268 }
269 
270 static unsigned
271 sha256_finish(__ops_hash_t *hash, unsigned char *out)
272 {
273 	SHA256_Final(out, hash->data);
274 	if (__ops_get_debug_level(__FILE__)) {
275 		unsigned        i;
276 
277 		(void) fprintf(stderr, "***\n***\nsha1_finish\n***\n");
278 		for (i = 0; i < SHA256_DIGEST_LENGTH; i++)
279 			(void) fprintf(stderr, "0x%02x ", out[i]);
280 		(void) fprintf(stderr, "\n");
281 	}
282 	free(hash->data);
283 	hash->data = NULL;
284 	return SHA256_DIGEST_LENGTH;
285 }
286 
287 static const __ops_hash_t sha256 = {
288 	OPS_HASH_SHA256,
289 	SHA256_DIGEST_LENGTH,
290 	"SHA256",
291 	sha256_init,
292 	sha256_add,
293 	sha256_finish,
294 	NULL
295 };
296 
297 void
298 __ops_hash_sha256(__ops_hash_t *hash)
299 {
300 	*hash = sha256;
301 }
302 
303 /*
304  * SHA384
305  */
306 static int
307 sha384_init(__ops_hash_t *hash)
308 {
309 	if (__ops_get_debug_level(__FILE__)) {
310 		(void) fprintf(stderr, "***\n***\nsha384_init\n***\n");
311 	}
312 	if (hash->data) {
313 		(void) fprintf(stderr, "sha384_init: hash data non-null\n");
314 	}
315 	if ((hash->data = calloc(1, sizeof(SHA512_CTX))) == NULL) {
316 		(void) fprintf(stderr, "sha512_init: bad alloc\n");
317 		return 0;
318 	}
319 	SHA384_Init(hash->data);
320 	return 1;
321 }
322 
323 static void
324 sha384_add(__ops_hash_t *hash, const unsigned char *data, unsigned length)
325 {
326 	if (__ops_get_debug_level(__FILE__)) {
327 		unsigned int    i;
328 
329 		(void) fprintf(stderr, "adding %u to hash:\n ", length);
330 		for (i = 0; i < length; i++) {
331 			(void) fprintf(stderr, "0x%02x ", data[i]);
332 			if (!((i + 1) % 16))
333 				(void) fprintf(stderr, "\n");
334 			else if (!((i + 1) % 8))
335 				(void) fprintf(stderr, "  ");
336 		}
337 		(void) fprintf(stderr, "\n");
338 	}
339 	SHA384_Update(hash->data, data, length);
340 }
341 
342 static unsigned
343 sha384_finish(__ops_hash_t *hash, unsigned char *out)
344 {
345 	SHA384_Final(out, hash->data);
346 	if (__ops_get_debug_level(__FILE__)) {
347 		unsigned        i;
348 
349 		(void) fprintf(stderr, "***\n***\nsha1_finish\n***\n");
350 		for (i = 0; i < SHA384_DIGEST_LENGTH; i++)
351 			(void) fprintf(stderr, "0x%02x ", out[i]);
352 		(void) fprintf(stderr, "\n");
353 	}
354 	free(hash->data);
355 	hash->data = NULL;
356 	return SHA384_DIGEST_LENGTH;
357 }
358 
359 static const __ops_hash_t sha384 = {
360 	OPS_HASH_SHA384,
361 	SHA384_DIGEST_LENGTH,
362 	"SHA384",
363 	sha384_init,
364 	sha384_add,
365 	sha384_finish,
366 	NULL
367 };
368 
369 void
370 __ops_hash_sha384(__ops_hash_t *hash)
371 {
372 	*hash = sha384;
373 }
374 
375 /*
376  * SHA512
377  */
378 static int
379 sha512_init(__ops_hash_t *hash)
380 {
381 	if (__ops_get_debug_level(__FILE__)) {
382 		(void) fprintf(stderr, "***\n***\nsha512_init\n***\n");
383 	}
384 	if (hash->data) {
385 		(void) fprintf(stderr, "sha512_init: hash data non-null\n");
386 	}
387 	if ((hash->data = calloc(1, sizeof(SHA512_CTX))) == NULL) {
388 		(void) fprintf(stderr, "sha512_init: bad alloc\n");
389 		return 0;
390 	}
391 	SHA512_Init(hash->data);
392 	return 1;
393 }
394 
395 static void
396 sha512_add(__ops_hash_t *hash, const unsigned char *data, unsigned length)
397 {
398 	if (__ops_get_debug_level(__FILE__)) {
399 		unsigned int    i;
400 
401 		(void) fprintf(stderr, "adding %u to hash:\n ", length);
402 		for (i = 0; i < length; i++) {
403 			(void) fprintf(stderr, "0x%02x ", data[i]);
404 			if (!((i + 1) % 16))
405 				(void) fprintf(stderr, "\n");
406 			else if (!((i + 1) % 8))
407 				(void) fprintf(stderr, "  ");
408 		}
409 		(void) fprintf(stderr, "\n");
410 	}
411 	SHA512_Update(hash->data, data, length);
412 }
413 
414 static unsigned
415 sha512_finish(__ops_hash_t *hash, unsigned char *out)
416 {
417 	SHA512_Final(out, hash->data);
418 	if (__ops_get_debug_level(__FILE__)) {
419 		unsigned        i;
420 
421 		(void) fprintf(stderr, "***\n***\nsha1_finish\n***\n");
422 		for (i = 0; i < SHA512_DIGEST_LENGTH; i++)
423 			(void) fprintf(stderr, "0x%02x ", out[i]);
424 		(void) fprintf(stderr, "\n");
425 	}
426 	free(hash->data);
427 	hash->data = NULL;
428 	return SHA512_DIGEST_LENGTH;
429 }
430 
431 static const __ops_hash_t sha512 = {
432 	OPS_HASH_SHA512,
433 	SHA512_DIGEST_LENGTH,
434 	"SHA512",
435 	sha512_init,
436 	sha512_add,
437 	sha512_finish,
438 	NULL
439 };
440 
441 void
442 __ops_hash_sha512(__ops_hash_t *hash)
443 {
444 	*hash = sha512;
445 }
446 
447 /*
448  * SHA224
449  */
450 
451 static int
452 sha224_init(__ops_hash_t *hash)
453 {
454 	if (__ops_get_debug_level(__FILE__)) {
455 		(void) fprintf(stderr, "***\n***\nsha1_init\n***\n");
456 	}
457 	if (hash->data) {
458 		(void) fprintf(stderr, "sha224_init: hash data non-null\n");
459 	}
460 	if ((hash->data = calloc(1, sizeof(SHA256_CTX))) == NULL) {
461 		(void) fprintf(stderr, "sha256_init: bad alloc\n");
462 		return 0;
463 	}
464 	SHA224_Init(hash->data);
465 	return 1;
466 }
467 
468 static void
469 sha224_add(__ops_hash_t *hash, const unsigned char *data, unsigned length)
470 {
471 	if (__ops_get_debug_level(__FILE__)) {
472 		unsigned int    i;
473 
474 		(void) fprintf(stderr, "adding %u to hash:\n ", length);
475 		for (i = 0; i < length; i++) {
476 			(void) fprintf(stderr, "0x%02x ", data[i]);
477 			if (!((i + 1) % 16))
478 				(void) fprintf(stderr, "\n");
479 			else if (!((i + 1) % 8))
480 				(void) fprintf(stderr, "  ");
481 		}
482 		(void) fprintf(stderr, "\n");
483 	}
484 	SHA224_Update(hash->data, data, length);
485 }
486 
487 static unsigned
488 sha224_finish(__ops_hash_t *hash, unsigned char *out)
489 {
490 	SHA224_Final(out, hash->data);
491 	if (__ops_get_debug_level(__FILE__)) {
492 		unsigned        i;
493 
494 		(void) fprintf(stderr, "***\n***\nsha1_finish\n***\n");
495 		for (i = 0; i < SHA224_DIGEST_LENGTH; i++)
496 			(void) fprintf(stderr, "0x%02x ", out[i]);
497 		(void) fprintf(stderr, "\n");
498 	}
499 	free(hash->data);
500 	hash->data = NULL;
501 	return SHA224_DIGEST_LENGTH;
502 }
503 
504 static const __ops_hash_t sha224 = {
505 	OPS_HASH_SHA224,
506 	SHA224_DIGEST_LENGTH,
507 	"SHA224",
508 	sha224_init,
509 	sha224_add,
510 	sha224_finish,
511 	NULL
512 };
513 
514 void
515 __ops_hash_sha224(__ops_hash_t *hash)
516 {
517 	*hash = sha224;
518 }
519 
520 unsigned
521 __ops_dsa_verify(const unsigned char *hash, size_t hash_length,
522 	       const __ops_dsa_sig_t *sig,
523 	       const __ops_dsa_pubkey_t *dsa)
524 {
525 	unsigned	qlen;
526 	DSA_SIG        *osig;
527 	DSA            *odsa;
528 	int             ret;
529 
530 	osig = DSA_SIG_new();
531 	osig->r = sig->r;
532 	osig->s = sig->s;
533 
534 	odsa = DSA_new();
535 	odsa->p = dsa->p;
536 	odsa->q = dsa->q;
537 	odsa->g = dsa->g;
538 	odsa->pub_key = dsa->y;
539 
540 	if (__ops_get_debug_level(__FILE__)) {
541 		unsigned        i;
542 
543 		(void) fprintf(stderr, "hash passed in:\n");
544 		for (i = 0; i < hash_length; i++) {
545 			(void) fprintf(stderr, "%02x ", hash[i]);
546 		}
547 		(void) fprintf(stderr, "\n");
548 		printf("hash_length=%" PRIsize "d\n", hash_length);
549 		printf("Q=%d\n", BN_num_bytes(odsa->q));
550 	}
551 	/* XXX - Flexelint -  Info 732: Loss of sign (assignment) (int to unsigned int) */
552 	if ((qlen = BN_num_bytes(odsa->q)) < hash_length) {
553 		hash_length = qlen;
554 	}
555 	ret = DSA_do_verify(hash, (int)hash_length, osig, odsa);
556 	if (__ops_get_debug_level(__FILE__)) {
557 		(void) fprintf(stderr, "ret=%d\n", ret);
558 	}
559 	if (ret < 0) {
560 		(void) fprintf(stderr, "__ops_dsa_verify: DSA verification\n");
561 		return 0;
562 	}
563 
564 	odsa->p = odsa->q = odsa->g = odsa->pub_key = NULL;
565 	DSA_free(odsa);
566 
567 	osig->r = osig->s = NULL;
568 	DSA_SIG_free(osig);
569 
570 	return (unsigned)ret;
571 }
572 
573 /**
574    \ingroup Core_Crypto
575    \brief Recovers message digest from the signature
576    \param out Where to write decrypted data to
577    \param in Encrypted data
578    \param length Length of encrypted data
579    \param pubkey RSA public key
580    \return size of recovered message digest
581 */
582 int
583 __ops_rsa_public_decrypt(unsigned char *out,
584 			const unsigned char *in,
585 			size_t length,
586 			const __ops_rsa_pubkey_t *pubkey)
587 {
588 	RSA            *orsa;
589 	int             n;
590 
591 	orsa = RSA_new();
592 	orsa->n = pubkey->n;
593 	orsa->e = pubkey->e;
594 
595 	n = RSA_public_decrypt((int)length, in, out, orsa, RSA_NO_PADDING);
596 
597 	orsa->n = orsa->e = NULL;
598 	RSA_free(orsa);
599 
600 	return n;
601 }
602 
603 /**
604    \ingroup Core_Crypto
605    \brief Signs data with RSA
606    \param out Where to write signature
607    \param in Data to sign
608    \param length Length of data
609    \param seckey RSA secret key
610    \param pubkey RSA public key
611    \return number of bytes decrypted
612 */
613 int
614 __ops_rsa_private_encrypt(unsigned char *out,
615 			const unsigned char *in,
616 			size_t length,
617 			const __ops_rsa_seckey_t *seckey,
618 			const __ops_rsa_pubkey_t *pubkey)
619 {
620 	RSA            *orsa;
621 	int             n;
622 
623 	orsa = RSA_new();
624 	orsa->n = pubkey->n;	/* XXX: do we need n? */
625 	orsa->d = seckey->d;
626 	orsa->p = seckey->q;
627 	orsa->q = seckey->p;
628 
629 	/* debug */
630 	orsa->e = pubkey->e;
631 	/* If this isn't set, it's very likely that the programmer hasn't */
632 	/* decrypted the secret key. RSA_check_key segfaults in that case. */
633 	/* Use __ops_decrypt_seckey() to do that. */
634 	if (orsa->d == NULL) {
635 		(void) fprintf(stderr, "orsa is not set\n");
636 		return 0;
637 	}
638 	if (RSA_check_key(orsa) != 1) {
639 		(void) fprintf(stderr, "RSA_check_key is not set\n");
640 		return 0;
641 	}
642 	/* end debug */
643 
644 	n = RSA_private_encrypt((int)length, in, out, orsa, RSA_NO_PADDING);
645 
646 	orsa->n = orsa->d = orsa->p = orsa->q = NULL;
647 	RSA_free(orsa);
648 
649 	return n;
650 }
651 
652 /**
653 \ingroup Core_Crypto
654 \brief Decrypts RSA-encrypted data
655 \param out Where to write the plaintext
656 \param in Encrypted data
657 \param length Length of encrypted data
658 \param seckey RSA secret key
659 \param pubkey RSA public key
660 \return size of recovered plaintext
661 */
662 int
663 __ops_rsa_private_decrypt(unsigned char *out,
664 			const unsigned char *in,
665 			size_t length,
666 			const __ops_rsa_seckey_t *seckey,
667 			const __ops_rsa_pubkey_t *pubkey)
668 {
669 	RSA            *keypair;
670 	int             n;
671 	char            errbuf[1024];
672 
673 	keypair = RSA_new();
674 	keypair->n = pubkey->n;	/* XXX: do we need n? */
675 	keypair->d = seckey->d;
676 	keypair->p = seckey->q;
677 	keypair->q = seckey->p;
678 
679 	/* debug */
680 	keypair->e = pubkey->e;
681 	if (RSA_check_key(keypair) != 1) {
682 		(void) fprintf(stderr, "RSA_check_key is not set\n");
683 		return 0;
684 	}
685 	/* end debug */
686 
687 	n = RSA_private_decrypt((int)length, in, out, keypair, RSA_NO_PADDING);
688 
689 	if (__ops_get_debug_level(__FILE__)) {
690 		printf("__ops_rsa_private_decrypt: n=%d\n",n);
691 	}
692 
693 	errbuf[0] = '\0';
694 	if (n == -1) {
695 		unsigned long   err = ERR_get_error();
696 
697 		ERR_error_string(err, &errbuf[0]);
698 		(void) fprintf(stderr, "openssl error : %s\n", errbuf);
699 	}
700 	keypair->n = keypair->d = keypair->p = keypair->q = NULL;
701 	RSA_free(keypair);
702 
703 	return n;
704 }
705 
706 /**
707    \ingroup Core_Crypto
708    \brief RSA-encrypts data
709    \param out Where to write the encrypted data
710    \param in Plaintext
711    \param length Size of plaintext
712    \param pubkey RSA Public Key
713 */
714 int
715 __ops_rsa_public_encrypt(unsigned char *out,
716 			const unsigned char *in,
717 			size_t length,
718 			const __ops_rsa_pubkey_t *pubkey)
719 {
720 	RSA            *orsa;
721 	int             n;
722 
723 	/* printf("__ops_rsa_public_encrypt: length=%ld\n", length); */
724 
725 	orsa = RSA_new();
726 	orsa->n = pubkey->n;
727 	orsa->e = pubkey->e;
728 
729 	/* printf("len: %ld\n", length); */
730 	/* __ops_print_bn("n: ", orsa->n); */
731 	/* __ops_print_bn("e: ", orsa->e); */
732 	n = RSA_public_encrypt((int)length, in, out, orsa, RSA_NO_PADDING);
733 
734 	if (n == -1) {
735 		BIO            *fd_out;
736 
737 		fd_out = BIO_new_fd(fileno(stderr), BIO_NOCLOSE);
738 		ERR_print_errors(fd_out);
739 	}
740 	orsa->n = orsa->e = NULL;
741 	RSA_free(orsa);
742 
743 	return n;
744 }
745 
746 /**
747    \ingroup Core_Crypto
748    \brief initialises openssl
749    \note Would usually call __ops_init() instead
750    \sa __ops_init()
751 */
752 void
753 __ops_crypto_init(void)
754 {
755 #ifdef DMALLOC
756 	CRYPTO_malloc_debug_init();
757 	CRYPTO_dbg_set_options(V_CRYPTO_MDEBUG_ALL);
758 	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
759 #endif
760 }
761 
762 /**
763    \ingroup Core_Crypto
764    \brief Finalise openssl
765    \note Would usually call __ops_finish() instead
766    \sa __ops_finish()
767 */
768 void
769 __ops_crypto_finish(void)
770 {
771 	CRYPTO_cleanup_all_ex_data();
772 	ERR_remove_state((unsigned long)0);
773 #ifdef DMALLOC
774 	CRYPTO_mem_leaks_fp(stderr);
775 #endif
776 }
777 
778 /**
779    \ingroup Core_Hashes
780    \brief Get Hash name
781    \param hash Hash struct
782    \return Hash name
783 */
784 const char     *
785 __ops_text_from_hash(__ops_hash_t *hash)
786 {
787 	return hash->name;
788 }
789 
790 /**
791  \ingroup HighLevel_KeyGenerate
792  \brief Generates an RSA keypair
793  \param numbits Modulus size
794  \param e Public Exponent
795  \param keydata Pointer to keydata struct to hold new key
796  \return 1 if key generated successfully; otherwise 0
797  \note It is the caller's responsibility to call __ops_keydata_free(keydata)
798 */
799 static unsigned
800 rsa_generate_keypair(__ops_key_t *keydata,
801 			const int numbits,
802 			const unsigned long e)
803 {
804 	__ops_seckey_t *seckey;
805 	RSA            *rsa;
806 	BN_CTX         *ctx;
807 	__ops_output_t *output;
808 	__ops_memory_t   *mem;
809 
810 	ctx = BN_CTX_new();
811 	__ops_keydata_init(keydata, OPS_PTAG_CT_SECRET_KEY);
812 	seckey = __ops_get_writable_seckey(keydata);
813 
814 	/* generate the key pair */
815 
816 	rsa = RSA_generate_key(numbits, e, NULL, NULL);
817 
818 	/* populate __ops key from ssl key */
819 
820 	seckey->pubkey.version = OPS_V4;
821 	seckey->pubkey.birthtime = time(NULL);
822 	seckey->pubkey.days_valid = 0;
823 	seckey->pubkey.alg = OPS_PKA_RSA;
824 
825 	seckey->pubkey.key.rsa.n = BN_dup(rsa->n);
826 	seckey->pubkey.key.rsa.e = BN_dup(rsa->e);
827 
828 	seckey->s2k_usage = OPS_S2KU_ENCRYPTED_AND_HASHED;
829 	seckey->s2k_specifier = OPS_S2KS_SALTED;
830 	/* seckey->s2k_specifier=OPS_S2KS_SIMPLE; */
831 	seckey->alg = OPS_SA_CAST5;	/* \todo make param */
832 	seckey->hash_alg = OPS_HASH_SHA1;	/* \todo make param */
833 	seckey->octetc = 0;
834 	seckey->checksum = 0;
835 
836 	seckey->key.rsa.d = BN_dup(rsa->d);
837 	seckey->key.rsa.p = BN_dup(rsa->p);
838 	seckey->key.rsa.q = BN_dup(rsa->q);
839 	seckey->key.rsa.u = BN_mod_inverse(NULL, rsa->p, rsa->q, ctx);
840 	if (seckey->key.rsa.u == NULL) {
841 		(void) fprintf(stderr, "seckey->key.rsa.u is NULL\n");
842 		return 0;
843 	}
844 	BN_CTX_free(ctx);
845 
846 	RSA_free(rsa);
847 
848 	__ops_keyid(keydata->key_id, OPS_KEY_ID_SIZE,
849 			&keydata->key.seckey.pubkey);
850 	__ops_fingerprint(&keydata->fingerprint, &keydata->key.seckey.pubkey);
851 
852 	/* Generate checksum */
853 
854 	output = NULL;
855 	mem = NULL;
856 
857 	__ops_setup_memory_write(&output, &mem, 128);
858 
859 	__ops_push_checksum_writer(output, seckey);
860 
861 	switch (seckey->pubkey.alg) {
862 		/* case OPS_PKA_DSA: */
863 		/* return __ops_write_mpi(output, key->key.dsa.x); */
864 
865 	case OPS_PKA_RSA:
866 	case OPS_PKA_RSA_ENCRYPT_ONLY:
867 	case OPS_PKA_RSA_SIGN_ONLY:
868 		if (!__ops_write_mpi(output, seckey->key.rsa.d) ||
869 		    !__ops_write_mpi(output, seckey->key.rsa.p) ||
870 		    !__ops_write_mpi(output, seckey->key.rsa.q) ||
871 		    !__ops_write_mpi(output, seckey->key.rsa.u)) {
872 			return 0;
873 		}
874 		break;
875 
876 		/* case OPS_PKA_ELGAMAL: */
877 		/* return __ops_write_mpi(output, key->key.elgamal.x); */
878 
879 	default:
880 		(void) fprintf(stderr, "Bad seckey->pubkey.alg\n");
881 		return 0;
882 	}
883 
884 	/* close rather than pop, since its the only one on the stack */
885 	__ops_writer_close(output);
886 	__ops_teardown_memory_write(output, mem);
887 
888 	/* should now have checksum in seckey struct */
889 
890 	/* test */
891 	if (__ops_get_debug_level(__FILE__)) {
892 		test_seckey(seckey);
893 	}
894 
895 	return 1;
896 }
897 
898 /**
899  \ingroup HighLevel_KeyGenerate
900  \brief Creates a self-signed RSA keypair
901  \param numbits Modulus size
902  \param e Public Exponent
903  \param userid User ID
904  \return The new keypair or NULL
905 
906  \note It is the caller's responsibility to call __ops_keydata_free(keydata)
907  \sa rsa_generate_keypair()
908  \sa __ops_keydata_free()
909 */
910 __ops_key_t  *
911 __ops_rsa_new_selfsign_key(const int numbits,
912 				const unsigned long e,
913 				__ops_userid_t *userid)
914 {
915 	__ops_key_t  *keydata;
916 
917 	keydata = __ops_keydata_new();
918 	if (!rsa_generate_keypair(keydata, numbits, e) ||
919 	    !__ops_add_selfsigned_userid(keydata, userid)) {
920 		__ops_keydata_free(keydata);
921 		return NULL;
922 	}
923 	return keydata;
924 }
925 
926 DSA_SIG        *
927 __ops_dsa_sign(unsigned char *hashbuf,
928 		unsigned hashsize,
929 		const __ops_dsa_seckey_t *secdsa,
930 		const __ops_dsa_pubkey_t *pubdsa)
931 {
932 	DSA_SIG        *dsasig;
933 	DSA            *odsa;
934 
935 	odsa = DSA_new();
936 	odsa->p = pubdsa->p;
937 	odsa->q = pubdsa->q;
938 	odsa->g = pubdsa->g;
939 	odsa->pub_key = pubdsa->y;
940 	odsa->priv_key = secdsa->x;
941 
942 	dsasig = DSA_do_sign(hashbuf, (int)hashsize, odsa);
943 
944 	odsa->p = odsa->q = odsa->g = odsa->pub_key = odsa->priv_key = NULL;
945 	DSA_free(odsa);
946 
947 	return dsasig;
948 }
949 
950 int
951 openssl_read_pem_seckey(const char *f, __ops_key_t *key, const char *type, int verbose)
952 {
953 	FILE	*fp;
954 	DSA	*dsa;
955 	RSA	*rsa;
956 	int	 ok;
957 
958 	if ((fp = fopen(f, "r")) == NULL) {
959 		if (verbose) {
960 			(void) fprintf(stderr, "can't open '%s'\n", f);
961 		}
962 		return 0;
963 	}
964 	ok = 1;
965 	if (strcmp(type, "ssh-rsa") == 0) {
966 		if ((rsa = PEM_read_RSAPrivateKey(fp, NULL, NULL, NULL)) == NULL) {
967 			ok = 0;
968 		} else {
969 			key->key.seckey.key.rsa.d = rsa->d;
970 			key->key.seckey.key.rsa.p = rsa->p;
971 			key->key.seckey.key.rsa.q = rsa->q;
972 			key->key.seckey.key.rsa.d = rsa->d;
973 		}
974 	} else if (strcmp(type, "ssh-dss") == 0) {
975 		if ((dsa = PEM_read_DSAPrivateKey(fp, NULL, NULL, NULL)) == NULL) {
976 			ok = 0;
977 		} else {
978 			key->key.seckey.key.dsa.x = dsa->priv_key;
979 		}
980 	} else {
981 		ok = 0;
982 	}
983 	(void) fclose(fp);
984 	return ok;
985 }
986