xref: /openbsd-src/lib/libkeynote/signature.c (revision c1a45aed656e7d5627c30c92421893a76f370ccb)
1 /* $OpenBSD: signature.c,v 1.29 2022/01/14 09:08:03 tb Exp $ */
2 /*
3  * The author of this code is Angelos D. Keromytis (angelos@dsl.cis.upenn.edu)
4  *
5  * This code was written by Angelos D. Keromytis in Philadelphia, PA, USA,
6  * in April-May 1998
7  *
8  * Copyright (C) 1998, 1999 by Angelos D. Keromytis.
9  *
10  * Permission to use, copy, and modify this software with or without fee
11  * is hereby granted, provided that this entire notice is included in
12  * all copies of any software which is or includes a copy or
13  * modification of this software.
14  *
15  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
16  * IMPLIED WARRANTY. IN PARTICULAR, THE AUTHORS MAKES NO
17  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
18  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
19  * PURPOSE.
20  */
21 
22 /*
23  * Support for X509 keys and signing added by Ben Laurie <ben@algroup.co.uk>
24  * 3 May 1999
25  */
26 
27 #include <sys/types.h>
28 
29 #include <limits.h>
30 #include <regex.h>
31 #include <stdlib.h>
32 #include <stdio.h>
33 #include <string.h>
34 
35 #include <openssl/dsa.h>
36 #include <openssl/md5.h>
37 #include <openssl/pem.h>
38 #include <openssl/rsa.h>
39 #include <openssl/sha.h>
40 #include <openssl/x509.h>
41 
42 #include "keynote.h"
43 #include "assertion.h"
44 #include "signature.h"
45 
46 static const char hextab[] = {
47      '0', '1', '2', '3', '4', '5', '6', '7',
48      '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
49 };
50 
51 /*
52  * Actual conversion to hex.
53  */
54 static void
55 bin2hex(unsigned char *data, unsigned char *buffer, int len)
56 {
57     int off = 0;
58 
59     while(len > 0)
60     {
61 	buffer[off++] = hextab[*data >> 4];
62 	buffer[off++] = hextab[*data & 0xF];
63 	data++;
64 	len--;
65     }
66 }
67 
68 /*
69  * Encode a binary string with hex encoding. Return 0 on success.
70  */
71 int
72 kn_encode_hex(unsigned char *buf, char **dest, int len)
73 {
74     keynote_errno = 0;
75     if (dest == NULL)
76     {
77 	keynote_errno = ERROR_SYNTAX;
78 	return -1;
79     }
80 
81     *dest = calloc(2 * len + 1, sizeof(char));
82     if (*dest == NULL)
83     {
84 	keynote_errno = ERROR_MEMORY;
85 	return -1;
86     }
87 
88     bin2hex(buf, *dest, len);
89     return 0;
90 }
91 
92 /*
93  * Decode a hex encoding. Return 0 on success. The second argument
94  * will be half as large as the first.
95  */
96 int
97 kn_decode_hex(char *hex, char **dest)
98 {
99     int i, decodedlen;
100     char ptr[3];
101 
102     keynote_errno = 0;
103     if (dest == NULL)
104     {
105 	keynote_errno = ERROR_SYNTAX;
106 	return -1;
107     }
108 
109     if (strlen(hex) % 2)			/* Should be even */
110     {
111 	keynote_errno = ERROR_SYNTAX;
112 	return -1;
113     }
114 
115     decodedlen = strlen(hex) / 2;
116     *dest = calloc(decodedlen, sizeof(char));
117     if (*dest == NULL)
118     {
119 	keynote_errno = ERROR_MEMORY;
120 	return -1;
121     }
122 
123     ptr[2] = '\0';
124     for (i = 0; i < decodedlen; i++)
125     {
126 	ptr[0] = hex[2 * i];
127 	ptr[1] = hex[(2 * i) + 1];
128       	(*dest)[i] = (unsigned char) strtoul(ptr, NULL, 16);
129     }
130 
131     return 0;
132 }
133 
134 void
135 keynote_free_key(void *key, int type)
136 {
137     if (key == NULL)
138       return;
139 
140     /* DSA keys */
141     if (type == KEYNOTE_ALGORITHM_DSA)
142     {
143 	DSA_free(key);
144 	return;
145     }
146 
147     /* RSA keys */
148     if (type == KEYNOTE_ALGORITHM_RSA)
149     {
150 	RSA_free(key);
151 	return;
152     }
153 
154     /* X509 keys */
155     if (type == KEYNOTE_ALGORITHM_X509)
156     {
157 	RSA_free(key); /* RSA-specific */
158 	return;
159     }
160 
161     /* BINARY keys */
162     if (type == KEYNOTE_ALGORITHM_BINARY)
163     {
164 	free(((struct keynote_binary *) key)->bn_key);
165 	free(key);
166 	return;
167     }
168 
169     /* Catch-all case */
170     if (type == KEYNOTE_ALGORITHM_NONE)
171       free(key);
172 }
173 
174 /*
175  * Map a signature to an algorithm. Return algorithm number (defined in
176  * keynote.h), or KEYNOTE_ALGORITHM_NONE if unknown.
177  * Also return in the second, third and fourth arguments the digest
178  * algorithm, ASCII and internal encodings respectively.
179  */
180 static int
181 keynote_get_sig_algorithm(char *sig, int *hash, int *enc, int *internal)
182 {
183     if (sig == NULL)
184       return KEYNOTE_ALGORITHM_NONE;
185 
186     if (!strncasecmp(SIG_DSA_SHA1_HEX, sig, SIG_DSA_SHA1_HEX_LEN))
187     {
188 	*hash = KEYNOTE_HASH_SHA1;
189 	*enc = ENCODING_HEX;
190 	*internal = INTERNAL_ENC_ASN1;
191 	return KEYNOTE_ALGORITHM_DSA;
192     }
193 
194     if (!strncasecmp(SIG_DSA_SHA1_BASE64, sig, SIG_DSA_SHA1_BASE64_LEN))
195     {
196 	*hash = KEYNOTE_HASH_SHA1;
197 	*enc = ENCODING_BASE64;
198 	*internal = INTERNAL_ENC_ASN1;
199 	return KEYNOTE_ALGORITHM_DSA;
200     }
201 
202     if (!strncasecmp(SIG_RSA_MD5_PKCS1_HEX, sig, SIG_RSA_MD5_PKCS1_HEX_LEN))
203     {
204 	*hash = KEYNOTE_HASH_MD5;
205 	*enc = ENCODING_HEX;
206 	*internal = INTERNAL_ENC_PKCS1;
207 	return KEYNOTE_ALGORITHM_RSA;
208     }
209 
210     if (!strncasecmp(SIG_RSA_SHA1_PKCS1_HEX, sig, SIG_RSA_SHA1_PKCS1_HEX_LEN))
211     {
212 	*hash = KEYNOTE_HASH_SHA1;
213 	*enc = ENCODING_HEX;
214 	*internal = INTERNAL_ENC_PKCS1;
215 	return KEYNOTE_ALGORITHM_RSA;
216     }
217 
218     if (!strncasecmp(SIG_RSA_MD5_PKCS1_BASE64, sig,
219                      SIG_RSA_MD5_PKCS1_BASE64_LEN))
220     {
221 	*hash = KEYNOTE_HASH_MD5;
222 	*enc = ENCODING_BASE64;
223 	*internal = INTERNAL_ENC_PKCS1;
224 	return KEYNOTE_ALGORITHM_RSA;
225     }
226 
227     if (!strncasecmp(SIG_RSA_SHA1_PKCS1_BASE64, sig,
228                      SIG_RSA_SHA1_PKCS1_BASE64_LEN))
229     {
230 	*hash = KEYNOTE_HASH_SHA1;
231 	*enc = ENCODING_BASE64;
232 	*internal = INTERNAL_ENC_PKCS1;
233 	return KEYNOTE_ALGORITHM_RSA;
234     }
235 
236     if (!strncasecmp(SIG_X509_SHA1_BASE64, sig, SIG_X509_SHA1_BASE64_LEN))
237     {
238 	*hash = KEYNOTE_HASH_SHA1;
239 	*enc = ENCODING_BASE64;
240 	*internal = INTERNAL_ENC_ASN1;
241 	return KEYNOTE_ALGORITHM_X509;
242     }
243 
244     if (!strncasecmp(SIG_X509_SHA1_HEX, sig, SIG_X509_SHA1_HEX_LEN))
245     {
246 	*hash = KEYNOTE_HASH_SHA1;
247 	*enc = ENCODING_HEX;
248 	*internal = INTERNAL_ENC_ASN1;
249 	return KEYNOTE_ALGORITHM_X509;
250     }
251 
252     *hash = KEYNOTE_HASH_NONE;
253     *enc = ENCODING_NONE;
254     *internal = INTERNAL_ENC_NONE;
255     return KEYNOTE_ALGORITHM_NONE;
256 }
257 
258 /*
259  * Map a key to an algorithm. Return algorithm number (defined in
260  * keynote.h), or KEYNOTE_ALGORITHM_NONE if unknown.
261  * This latter is also a valid algorithm (for logical tags). Also return
262  * in the second and third arguments the ASCII and internal encodings.
263  */
264 int
265 keynote_get_key_algorithm(char *key, int *encoding, int *internalencoding)
266 {
267     if (!strncasecmp(DSA_HEX, key, DSA_HEX_LEN))
268     {
269 	*internalencoding = INTERNAL_ENC_ASN1;
270 	*encoding = ENCODING_HEX;
271 	return KEYNOTE_ALGORITHM_DSA;
272     }
273 
274     if (!strncasecmp(DSA_BASE64, key, DSA_BASE64_LEN))
275     {
276 	*internalencoding = INTERNAL_ENC_ASN1;
277 	*encoding = ENCODING_BASE64;
278 	return KEYNOTE_ALGORITHM_DSA;
279     }
280 
281     if (!strncasecmp(RSA_PKCS1_HEX, key, RSA_PKCS1_HEX_LEN))
282     {
283 	*internalencoding = INTERNAL_ENC_PKCS1;
284 	*encoding = ENCODING_HEX;
285 	return KEYNOTE_ALGORITHM_RSA;
286     }
287 
288     if (!strncasecmp(RSA_PKCS1_BASE64, key, RSA_PKCS1_BASE64_LEN))
289     {
290 	*internalencoding = INTERNAL_ENC_PKCS1;
291 	*encoding = ENCODING_BASE64;
292 	return KEYNOTE_ALGORITHM_RSA;
293     }
294 
295     if (!strncasecmp(X509_BASE64, key, X509_BASE64_LEN))
296     {
297 	*internalencoding = INTERNAL_ENC_ASN1;
298 	*encoding = ENCODING_BASE64;
299 	return KEYNOTE_ALGORITHM_X509;
300     }
301 
302     if (!strncasecmp(X509_HEX, key, X509_HEX_LEN))
303     {
304 	*internalencoding = INTERNAL_ENC_ASN1;
305 	*encoding = ENCODING_HEX;
306 	return KEYNOTE_ALGORITHM_X509;
307     }
308 
309     if (!strncasecmp(BINARY_HEX, key, BINARY_HEX_LEN))
310     {
311 	*internalencoding = INTERNAL_ENC_NONE;
312 	*encoding = ENCODING_HEX;
313 	return KEYNOTE_ALGORITHM_BINARY;
314     }
315 
316     if (!strncasecmp(BINARY_BASE64, key, BINARY_BASE64_LEN))
317     {
318 	*internalencoding = INTERNAL_ENC_NONE;
319 	*encoding = ENCODING_BASE64;
320 	return KEYNOTE_ALGORITHM_BINARY;
321     }
322 
323     *internalencoding = INTERNAL_ENC_NONE;
324     *encoding = ENCODING_NONE;
325     return KEYNOTE_ALGORITHM_NONE;
326 }
327 
328 /*
329  * Same as keynote_get_key_algorithm(), only verify that this is
330  * a private key (just look at the prefix).
331  */
332 static int
333 keynote_get_private_key_algorithm(char *key, int *encoding,
334 				  int *internalencoding)
335 {
336     if (strncasecmp(KEYNOTE_PRIVATE_KEY_PREFIX, key,
337 		    KEYNOTE_PRIVATE_KEY_PREFIX_LEN))
338     {
339 	*internalencoding = INTERNAL_ENC_NONE;
340 	*encoding = ENCODING_NONE;
341 	return KEYNOTE_ALGORITHM_NONE;
342     }
343 
344     return keynote_get_key_algorithm(key + KEYNOTE_PRIVATE_KEY_PREFIX_LEN,
345 				     encoding, internalencoding);
346 }
347 
348 /*
349  * Decode a string to a key. Return 0 on success.
350  */
351 int
352 kn_decode_key(struct keynote_deckey *dc, char *key, int keytype)
353 {
354     void *kk = NULL;
355     X509 *px509Cert;
356     EVP_PKEY *pPublicKey;
357     unsigned char *ptr = NULL, *decoded = NULL;
358     int encoding, internalencoding;
359     long len = 0;
360 
361     keynote_errno = 0;
362     if (keytype == KEYNOTE_PRIVATE_KEY)
363       dc->dec_algorithm = keynote_get_private_key_algorithm(key, &encoding,
364 							    &internalencoding);
365     else
366       dc->dec_algorithm = keynote_get_key_algorithm(key, &encoding,
367 						    &internalencoding);
368     if (dc->dec_algorithm == KEYNOTE_ALGORITHM_NONE)
369     {
370 	if ((dc->dec_key = strdup(key)) == NULL) {
371 	    keynote_errno = ERROR_MEMORY;
372 	    return -1;
373 	}
374 
375 	return 0;
376     }
377 
378     key = strchr(key, ':'); /* Move forward, to the Encoding. We're guaranteed
379 			    * to have a ':' character, since this is a key */
380     key++;
381 
382     /* Remove ASCII encoding */
383     switch (encoding)
384     {
385 	case ENCODING_NONE:
386 	    break;
387 
388 	case ENCODING_HEX:
389             len = strlen(key) / 2;
390 	    if (kn_decode_hex(key, (char **) &decoded) != 0)
391 	      return -1;
392 	    ptr = decoded;
393 	    break;
394 
395 	case ENCODING_BASE64:
396 	    len = strlen(key);
397 	    if (len % 4)  /* Base64 encoding must be a multiple of 4 */
398 	    {
399 		keynote_errno = ERROR_SYNTAX;
400 		return -1;
401 	    }
402 
403 	    len = 3 * (len / 4);
404 	    decoded = calloc(len, sizeof(unsigned char));
405 	    ptr = decoded;
406 	    if (decoded == NULL) {
407 		keynote_errno = ERROR_MEMORY;
408 		return -1;
409 	    }
410 
411 	    if ((len = kn_decode_base64(key, decoded, len)) == -1)
412 	      return -1;
413 	    break;
414 
415 	case ENCODING_NATIVE:
416 	    decoded = strdup(key);
417 	    if (decoded == NULL) {
418 		keynote_errno = ERROR_MEMORY;
419 		return -1;
420 	    }
421 	    len = strlen(key);
422 	    ptr = decoded;
423 	    break;
424 
425 	default:
426 	    keynote_errno = ERROR_SYNTAX;
427 	    return -1;
428     }
429 
430     /* DSA-HEX */
431     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_DSA) &&
432 	(internalencoding == INTERNAL_ENC_ASN1))
433     {
434 	dc->dec_key = DSA_new();
435 	if (dc->dec_key == NULL) {
436 	    keynote_errno = ERROR_MEMORY;
437 	    return -1;
438 	}
439 
440 	kk = dc->dec_key;
441 	if (keytype == KEYNOTE_PRIVATE_KEY)
442 	{
443 	    if (d2i_DSAPrivateKey((DSA **) &kk,(const unsigned char **) &decoded, len) == NULL) {
444 		free(ptr);
445 		DSA_free(kk);
446 		keynote_errno = ERROR_SYNTAX; /* Could be a memory error */
447 		return -1;
448 	    }
449 	}
450 	else
451 	{
452 	    if (d2i_DSAPublicKey((DSA **) &kk, (const unsigned char **) &decoded, len) == NULL) {
453 		free(ptr);
454 		DSA_free(kk);
455 		keynote_errno = ERROR_SYNTAX; /* Could be a memory error */
456 		return -1;
457 	    }
458 	}
459 
460 	free(ptr);
461 
462 	return 0;
463     }
464 
465     /* RSA-PKCS1-HEX */
466     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_RSA) &&
467         (internalencoding == INTERNAL_ENC_PKCS1))
468     {
469         dc->dec_key = RSA_new();
470         if (dc->dec_key == NULL) {
471             keynote_errno = ERROR_MEMORY;
472             return -1;
473         }
474 
475         kk = dc->dec_key;
476         if (keytype == KEYNOTE_PRIVATE_KEY)
477         {
478             if (d2i_RSAPrivateKey((RSA **) &kk, (const unsigned char **) &decoded, len) == NULL) {
479                 free(ptr);
480                 RSA_free(kk);
481                 keynote_errno = ERROR_SYNTAX; /* Could be a memory error */
482                 return -1;
483             }
484 	    if (RSA_blinding_on((RSA *) kk, NULL) != 1) {
485                 free(ptr);
486                 RSA_free(kk);
487                 keynote_errno = ERROR_MEMORY;
488                 return -1;
489 	    }
490         }
491         else
492         {
493             if (d2i_RSAPublicKey((RSA **) &kk, (const unsigned char **) &decoded, len) == NULL) {
494                 free(ptr);
495                 RSA_free(kk);
496                 keynote_errno = ERROR_SYNTAX; /* Could be a memory error */
497                 return -1;
498             }
499         }
500 
501         free(ptr);
502 
503         return 0;
504     }
505 
506     /* X509 Cert */
507     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_X509) &&
508 	(internalencoding == INTERNAL_ENC_ASN1) &&
509 	(keytype == KEYNOTE_PUBLIC_KEY))
510     {
511 	if ((px509Cert = X509_new()) == NULL) {
512 	    free(ptr);
513 	    keynote_errno = ERROR_MEMORY;
514 	    return -1;
515 	}
516 
517 	if(d2i_X509(&px509Cert, (const unsigned char **)&decoded, len) == NULL)
518 	{
519 	    free(ptr);
520 	    X509_free(px509Cert);
521 	    keynote_errno = ERROR_SYNTAX;
522 	    return -1;
523 	}
524 
525 	if ((pPublicKey = X509_get0_pubkey(px509Cert)) == NULL) {
526 	    free(ptr);
527 	    X509_free(px509Cert);
528 	    keynote_errno = ERROR_SYNTAX;
529 	    return -1;
530 	}
531 
532 	/* RSA-specific */
533 	dc->dec_key = EVP_PKEY_get0_RSA(pPublicKey);
534 	RSA_up_ref(dc->dec_key);
535 
536 	free(ptr);
537 	X509_free(px509Cert);
538 	return 0;
539     }
540 
541     /* BINARY keys */
542     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_BINARY) &&
543 	(internalencoding == INTERNAL_ENC_NONE))
544     {
545 	dc->dec_key = calloc(1, sizeof(struct keynote_binary));
546 	if (dc->dec_key == NULL)
547 	{
548 	    keynote_errno = ERROR_MEMORY;
549 	    return -1;
550 	}
551 
552 	((struct keynote_binary *) dc->dec_key)->bn_key = decoded;
553 	((struct keynote_binary *) dc->dec_key)->bn_len = len;
554 	return RESULT_TRUE;
555     }
556 
557     /* Add support for more algorithms here */
558 
559     free(ptr);
560 
561     /* This shouldn't ever be reached really */
562     keynote_errno = ERROR_SYNTAX;
563     return -1;
564 }
565 
566 /*
567  * Compare two keys for equality. Return RESULT_TRUE if equal,
568  * RESULT_FALSE otherwise.
569  */
570 int
571 kn_keycompare(void *key1, void *key2, int algorithm)
572 {
573     DSA *p1, *p2;
574     RSA *p3, *p4;
575     struct keynote_binary *bn1, *bn2;
576 
577     if (key1 == NULL || key2 == NULL)
578       return RESULT_FALSE;
579 
580     switch (algorithm)
581     {
582 	case KEYNOTE_ALGORITHM_NONE:
583 	    if (!strcmp(key1, key2))
584 	      return RESULT_TRUE;
585 	    else
586 	      return RESULT_FALSE;
587 
588 	case KEYNOTE_ALGORITHM_DSA:
589 	    p1 = (DSA *) key1;
590 	    p2 = (DSA *) key2;
591 	    if (!BN_cmp(DSA_get0_p(p1), DSA_get0_p(p2)) &&
592 		!BN_cmp(DSA_get0_q(p1), DSA_get0_q(p2)) &&
593 		!BN_cmp(DSA_get0_g(p1), DSA_get0_g(p2)) &&
594 		!BN_cmp(DSA_get0_pub_key(p1), DSA_get0_pub_key(p2)))
595 	      return RESULT_TRUE;
596 	    else
597 	      return RESULT_FALSE;
598 
599 	case KEYNOTE_ALGORITHM_X509:
600             p3 = (RSA *) key1;
601             p4 = (RSA *) key2;
602             if (!BN_cmp(RSA_get0_n(p3), RSA_get0_n(p4)) &&
603                 !BN_cmp(RSA_get0_e(p3), RSA_get0_e(p4)))
604               return RESULT_TRUE;
605             else
606 	      return RESULT_FALSE;
607 
608 	case KEYNOTE_ALGORITHM_RSA:
609             p3 = (RSA *) key1;
610             p4 = (RSA *) key2;
611             if (!BN_cmp(RSA_get0_n(p3), RSA_get0_n(p4)) &&
612                 !BN_cmp(RSA_get0_e(p3), RSA_get0_e(p4)))
613               return RESULT_TRUE;
614             else
615 	      return RESULT_FALSE;
616 
617 	case KEYNOTE_ALGORITHM_ELGAMAL:
618 	    /* Not supported yet */
619 	    return RESULT_FALSE;
620 
621 	case KEYNOTE_ALGORITHM_PGP:
622 	    /* Not supported yet */
623 	    return RESULT_FALSE;
624 
625 	case KEYNOTE_ALGORITHM_BINARY:
626 	    bn1 = (struct keynote_binary *) key1;
627 	    bn2 = (struct keynote_binary *) key2;
628 	    if ((bn1->bn_len == bn2->bn_len) &&
629 		!memcmp(bn1->bn_key, bn2->bn_key, bn1->bn_len))
630 	      return RESULT_TRUE;
631 	    else
632 	      return RESULT_FALSE;
633 
634 	default:
635 	    return RESULT_FALSE;
636     }
637 }
638 
639 /*
640  * Verify the signature on an assertion; return SIGRESULT_TRUE is
641  * success, SIGRESULT_FALSE otherwise.
642  */
643 int
644 keynote_sigverify_assertion(struct assertion *as)
645 {
646     int hashtype, enc, intenc, alg = KEYNOTE_ALGORITHM_NONE, hashlen = 0;
647     unsigned char *sig, *decoded = NULL, *ptr;
648     unsigned char res2[20];
649     SHA_CTX shscontext;
650     MD5_CTX md5context;
651     int len = 0;
652     DSA *dsa;
653     RSA *rsa;
654     if (as->as_signature == NULL ||
655 	as->as_startofsignature == NULL ||
656 	as->as_allbutsignature == NULL ||
657 	as->as_allbutsignature - as->as_startofsignature <= 0)
658       return SIGRESULT_FALSE;
659 
660     alg = keynote_get_sig_algorithm(as->as_signature, &hashtype, &enc,
661 				    &intenc);
662     if (alg == KEYNOTE_ALGORITHM_NONE)
663       return SIGRESULT_FALSE;
664 
665     /* Check for matching algorithms */
666     if ((alg != as->as_signeralgorithm) &&
667 	!((alg == KEYNOTE_ALGORITHM_RSA) &&
668 	  (as->as_signeralgorithm == KEYNOTE_ALGORITHM_X509)) &&
669 	!((alg == KEYNOTE_ALGORITHM_X509) &&
670 	  (as->as_signeralgorithm == KEYNOTE_ALGORITHM_RSA)))
671       return SIGRESULT_FALSE;
672 
673     sig = strchr(as->as_signature, ':');   /* Move forward to the Encoding. We
674 					   * are guaranteed to have a ':'
675 					   * character, since this is a valid
676 					   * signature */
677     sig++;
678 
679     switch (hashtype)
680     {
681 	case KEYNOTE_HASH_SHA1:
682 	    hashlen = 20;
683 	    memset(res2, 0, hashlen);
684 	    SHA1_Init(&shscontext);
685 	    SHA1_Update(&shscontext, as->as_startofsignature,
686 			as->as_allbutsignature - as->as_startofsignature);
687 	    SHA1_Update(&shscontext, as->as_signature,
688 			(char *) sig - as->as_signature);
689 	    SHA1_Final(res2, &shscontext);
690 	    break;
691 
692 	case KEYNOTE_HASH_MD5:
693 	    hashlen = 16;
694 	    memset(res2, 0, hashlen);
695 	    MD5_Init(&md5context);
696 	    MD5_Update(&md5context, as->as_startofsignature,
697 		       as->as_allbutsignature - as->as_startofsignature);
698 	    MD5_Update(&md5context, as->as_signature,
699 		       (char *) sig - as->as_signature);
700 	    MD5_Final(res2, &md5context);
701 	    break;
702 
703 	case KEYNOTE_HASH_NONE:
704 	    break;
705     }
706 
707     /* Remove ASCII encoding */
708     switch (enc)
709     {
710 	case ENCODING_NONE:
711 	    ptr = NULL;
712 	    break;
713 
714 	case ENCODING_HEX:
715 	    len = strlen(sig) / 2;
716 	    if (kn_decode_hex(sig, (char **) &decoded) != 0)
717 	      return -1;
718 	    ptr = decoded;
719 	    break;
720 
721 	case ENCODING_BASE64:
722 	    len = strlen(sig);
723 	    if (len % 4)  /* Base64 encoding must be a multiple of 4 */
724 	    {
725 		keynote_errno = ERROR_SYNTAX;
726 		return -1;
727 	    }
728 
729 	    len = 3 * (len / 4);
730 	    decoded = calloc(len, sizeof(unsigned char));
731 	    ptr = decoded;
732 	    if (decoded == NULL) {
733 		keynote_errno = ERROR_MEMORY;
734 		return -1;
735 	    }
736 
737 	    len = kn_decode_base64(sig, decoded, len);
738 	    if ((len == -1) || (len == 0) || (len == 1))
739 	      return -1;
740 	    break;
741 
742 	case ENCODING_NATIVE:
743 
744 	    if ((decoded = strdup(sig)) == NULL) {
745 		keynote_errno = ERROR_MEMORY;
746 		return -1;
747 	    }
748 	    len = strlen(sig);
749 	    ptr = decoded;
750 	    break;
751 
752 	default:
753 	    keynote_errno = ERROR_SYNTAX;
754 	    return -1;
755     }
756 
757     /* DSA */
758     if ((alg == KEYNOTE_ALGORITHM_DSA) && (intenc == INTERNAL_ENC_ASN1))
759     {
760 	dsa = (DSA *) as->as_authorizer;
761 	if (DSA_verify(0, res2, hashlen, decoded, len, dsa) == 1) {
762 	    free(ptr);
763 	    return SIGRESULT_TRUE;
764 	}
765     }
766     else /* RSA */
767       if ((alg == KEYNOTE_ALGORITHM_RSA) && (intenc == INTERNAL_ENC_PKCS1))
768       {
769           rsa = (RSA *) as->as_authorizer;
770           if (RSA_verify_ASN1_OCTET_STRING(RSA_PKCS1_PADDING, res2, hashlen,
771 					   decoded, len, rsa) == 1) {
772               free(ptr);
773               return SIGRESULT_TRUE;
774           }
775       }
776       else
777 	if ((alg == KEYNOTE_ALGORITHM_X509) && (intenc == INTERNAL_ENC_ASN1))
778 	{
779 	    /* RSA-specific */
780 	    rsa = (RSA *) as->as_authorizer;
781 	    if (RSA_verify(NID_shaWithRSAEncryption, res2, hashlen, decoded,
782 			   len, rsa) == 1) {
783 		free(ptr);
784 		return SIGRESULT_TRUE;
785 	    }
786 	}
787 
788     /* Handle more algorithms here */
789 
790     free(ptr);
791 
792     return SIGRESULT_FALSE;
793 }
794 
795 /*
796  * Sign an assertion.
797  */
798 static char *
799 keynote_sign_assertion(struct assertion *as, char *sigalg, void *key,
800 		       int keyalg, int verifyflag)
801 {
802     int slen, i, hashlen = 0, hashtype, alg, encoding, internalenc;
803     unsigned char *sig = NULL, *finalbuf = NULL;
804     unsigned char res2[LARGEST_HASH_SIZE], *sbuf = NULL;
805     BIO *biokey = NULL;
806     DSA *dsa = NULL;
807     RSA *rsa = NULL;
808     SHA_CTX shscontext;
809     MD5_CTX md5context;
810     int len;
811 
812     if (as->as_signature_string_s == NULL ||
813 	as->as_startofsignature == NULL ||
814 	as->as_allbutsignature == NULL ||
815 	as->as_allbutsignature - as->as_startofsignature <= 0 ||
816 	as->as_authorizer == NULL ||
817 	key == NULL ||
818 	as->as_signeralgorithm == KEYNOTE_ALGORITHM_NONE)
819     {
820 	keynote_errno = ERROR_SYNTAX;
821 	return NULL;
822     }
823 
824     alg = keynote_get_sig_algorithm(sigalg, &hashtype, &encoding,
825 				    &internalenc);
826     if (((alg != as->as_signeralgorithm) &&
827 	 !((alg == KEYNOTE_ALGORITHM_RSA) &&
828 	   (as->as_signeralgorithm == KEYNOTE_ALGORITHM_X509)) &&
829 	 !((alg == KEYNOTE_ALGORITHM_X509) &&
830 	   (as->as_signeralgorithm == KEYNOTE_ALGORITHM_RSA))) ||
831         ((alg != keyalg) &&
832 	 !((alg == KEYNOTE_ALGORITHM_RSA) &&
833 	   (keyalg == KEYNOTE_ALGORITHM_X509)) &&
834 	 !((alg == KEYNOTE_ALGORITHM_X509) &&
835 	   (keyalg == KEYNOTE_ALGORITHM_RSA))))
836     {
837 	keynote_errno = ERROR_SYNTAX;
838 	return NULL;
839     }
840 
841     sig = strchr(sigalg, ':');
842     if (sig == NULL)
843     {
844 	keynote_errno = ERROR_SYNTAX;
845 	return NULL;
846     }
847 
848     sig++;
849 
850     switch (hashtype)
851     {
852 	case KEYNOTE_HASH_SHA1:
853     	    hashlen = 20;
854 	    memset(res2, 0, hashlen);
855 	    SHA1_Init(&shscontext);
856 	    SHA1_Update(&shscontext, as->as_startofsignature,
857 			as->as_allbutsignature - as->as_startofsignature);
858 	    SHA1_Update(&shscontext, sigalg, (char *) sig - sigalg);
859 	    SHA1_Final(res2, &shscontext);
860 	    break;
861 
862 	case KEYNOTE_HASH_MD5:
863 	    hashlen = 16;
864 	    memset(res2, 0, hashlen);
865 	    MD5_Init(&md5context);
866 	    MD5_Update(&md5context, as->as_startofsignature,
867 		       as->as_allbutsignature - as->as_startofsignature);
868 	    MD5_Update(&md5context, sigalg, (char *) sig - sigalg);
869 	    MD5_Final(res2, &md5context);
870 	    break;
871 
872 	case KEYNOTE_HASH_NONE:
873 	    break;
874     }
875 
876     if ((alg == KEYNOTE_ALGORITHM_DSA) &&
877 	(hashtype == KEYNOTE_HASH_SHA1) &&
878 	(internalenc == INTERNAL_ENC_ASN1) &&
879 	((encoding == ENCODING_HEX) || (encoding == ENCODING_BASE64)))
880     {
881 	dsa = (DSA *) key;
882 	sbuf = calloc(DSA_size(dsa), sizeof(unsigned char));
883 	if (sbuf == NULL)
884 	{
885 	    keynote_errno = ERROR_MEMORY;
886 	    return NULL;
887 	}
888 
889 	if (DSA_sign(0, res2, hashlen, sbuf, &slen, dsa) <= 0)
890 	{
891 	    free(sbuf);
892 	    keynote_errno = ERROR_SYNTAX;
893 	    return NULL;
894 	}
895     }
896     else
897       if ((alg == KEYNOTE_ALGORITHM_RSA) &&
898           ((hashtype == KEYNOTE_HASH_SHA1) ||
899            (hashtype == KEYNOTE_HASH_MD5)) &&
900           (internalenc == INTERNAL_ENC_PKCS1) &&
901           ((encoding == ENCODING_HEX) || (encoding == ENCODING_BASE64)))
902       {
903           rsa = (RSA *) key;
904           sbuf = calloc(RSA_size(rsa), sizeof(unsigned char));
905           if (sbuf == NULL)
906           {
907               keynote_errno = ERROR_MEMORY;
908               return NULL;
909           }
910 
911           if (RSA_sign_ASN1_OCTET_STRING(RSA_PKCS1_PADDING, res2, hashlen,
912 					 sbuf, &slen, rsa) <= 0)
913           {
914               free(sbuf);
915               keynote_errno = ERROR_SYNTAX;
916               return NULL;
917           }
918       }
919     else
920       if ((alg == KEYNOTE_ALGORITHM_X509) &&
921 	  (hashtype == KEYNOTE_HASH_SHA1) &&
922 	  (internalenc == INTERNAL_ENC_ASN1))
923       {
924 	  if ((biokey = BIO_new(BIO_s_mem())) == NULL)
925 	  {
926 	      keynote_errno = ERROR_SYNTAX;
927 	      return NULL;
928 	  }
929 
930 	  if (BIO_write(biokey, key, strlen(key) + 1) <= 0)
931 	  {
932 	      BIO_free(biokey);
933 	      keynote_errno = ERROR_SYNTAX;
934 	      return NULL;
935 	  }
936 
937 	  /* RSA-specific */
938 	  rsa = (RSA *) PEM_read_bio_RSAPrivateKey(biokey, NULL, NULL, NULL);
939 	  if (rsa == NULL)
940 	  {
941 	      BIO_free(biokey);
942 	      keynote_errno = ERROR_SYNTAX;
943 	      return NULL;
944 	  }
945 
946 	  sbuf = calloc(RSA_size(rsa), sizeof(char));
947 	  if (sbuf == NULL)
948 	  {
949 	      BIO_free(biokey);
950 	      RSA_free(rsa);
951 	      keynote_errno = ERROR_MEMORY;
952 	      return NULL;
953 	  }
954 
955 	  if (RSA_sign(NID_shaWithRSAEncryption, res2, hashlen, sbuf, &slen,
956 		       rsa) <= 0)
957           {
958 	      BIO_free(biokey);
959 	      RSA_free(rsa);
960 	      free(sbuf);
961 	      keynote_errno = ERROR_SIGN_FAILURE;
962 	      return NULL;
963 	  }
964 
965 	  BIO_free(biokey);
966 	  RSA_free(rsa);
967       }
968       else /* Other algorithms here */
969       {
970 	  keynote_errno = ERROR_SYNTAX;
971 	  return NULL;
972       }
973 
974     /* ASCII encoding */
975     switch (encoding)
976     {
977 	case ENCODING_HEX:
978 	    i = kn_encode_hex(sbuf, (char **) &finalbuf, slen);
979 	    free(sbuf);
980 	    if (i != 0)
981 	      return NULL;
982 	    break;
983 
984 	case ENCODING_BASE64:
985 	    finalbuf = calloc(2 * slen, sizeof(unsigned char));
986 	    if (finalbuf == NULL)
987 	    {
988 		keynote_errno = ERROR_MEMORY;
989 		free(sbuf);
990 		return NULL;
991 	    }
992 
993 	    slen = kn_encode_base64(sbuf, slen, finalbuf, 2 * slen);
994 	    free(sbuf);
995 	    if (slen == -1) {
996 	      free(finalbuf);
997 	      return NULL;
998 	    }
999 	    break;
1000 
1001 	default:
1002 	    free(sbuf);
1003 	    keynote_errno = ERROR_SYNTAX;
1004 	    return NULL;
1005     }
1006 
1007     /* Replace as->as_signature */
1008     len = strlen(sigalg) + strlen(finalbuf) + 1;
1009     as->as_signature = calloc(len, sizeof(char));
1010     if (as->as_signature == NULL)
1011     {
1012 	free(finalbuf);
1013 	keynote_errno = ERROR_MEMORY;
1014 	return NULL;
1015     }
1016 
1017     /* Concatenate algorithm name and signature value */
1018     snprintf(as->as_signature, len, "%s%s", sigalg, finalbuf);
1019     free(finalbuf);
1020     finalbuf = as->as_signature;
1021 
1022     /* Verify the newly-created signature if requested */
1023     if (verifyflag)
1024     {
1025 	/* Do the signature verification */
1026 	if (keynote_sigverify_assertion(as) != SIGRESULT_TRUE)
1027 	{
1028 	    as->as_signature = NULL;
1029 	    free(finalbuf);
1030 	    if (keynote_errno == 0)
1031 	      keynote_errno = ERROR_SYNTAX;
1032 	    return NULL;
1033 	}
1034 
1035 	as->as_signature = NULL;
1036     }
1037     else
1038       as->as_signature = NULL;
1039 
1040     /* Everything ok */
1041     return (char *) finalbuf;
1042 }
1043 
1044 /*
1045  * Verify the signature on an assertion.
1046  */
1047 int
1048 kn_verify_assertion(char *buf, int len)
1049 {
1050     struct assertion *as;
1051     int res;
1052 
1053     keynote_errno = 0;
1054     as = keynote_parse_assertion(buf, len, ASSERT_FLAG_SIGVER);
1055     if (as == NULL)
1056       return -1;
1057 
1058     res = keynote_sigverify_assertion(as);
1059     keynote_free_assertion(as);
1060     return res;
1061 }
1062 
1063 /*
1064  * Produce the signature for an assertion.
1065  */
1066 char *
1067 kn_sign_assertion(char *buf, int buflen, char *key, char *sigalg, int vflag)
1068 {
1069     int i, alg, hashtype, encoding, internalenc;
1070     struct keynote_deckey dc;
1071     struct assertion *as;
1072     char *s, *sig;
1073 
1074     keynote_errno = 0;
1075     s = NULL;
1076 
1077     if (sigalg == NULL || buf == NULL || key == NULL)
1078     {
1079 	keynote_errno = ERROR_NOTFOUND;
1080 	return NULL;
1081     }
1082 
1083     if (sigalg[0] == '\0' || sigalg[strlen(sigalg) - 1] != ':')
1084     {
1085 	keynote_errno = ERROR_SYNTAX;
1086 	return NULL;
1087     }
1088 
1089     /* We're using a different format for X509 private keys, so... */
1090     alg = keynote_get_sig_algorithm(sigalg, &hashtype, &encoding,
1091 				    &internalenc);
1092     if (alg != KEYNOTE_ALGORITHM_X509)
1093     {
1094 	/* Parse the private key */
1095 	s = keynote_get_private_key(key);
1096 	if (s == NULL)
1097 	  return NULL;
1098 
1099 	/* Decode private key */
1100 	i = kn_decode_key(&dc, s, KEYNOTE_PRIVATE_KEY);
1101 	if (i == -1)
1102 	{
1103 	    free(s);
1104 	    return NULL;
1105 	}
1106     }
1107     else /* X509 private key */
1108     {
1109 	dc.dec_key = key;
1110 	dc.dec_algorithm = alg;
1111     }
1112 
1113     as = keynote_parse_assertion(buf, buflen, ASSERT_FLAG_SIGGEN);
1114     if (as == NULL)
1115     {
1116 	if (alg != KEYNOTE_ALGORITHM_X509)
1117 	{
1118 	    keynote_free_key(dc.dec_key, dc.dec_algorithm);
1119 	    free(s);
1120 	}
1121 	return NULL;
1122     }
1123 
1124     sig = keynote_sign_assertion(as, sigalg, dc.dec_key, dc.dec_algorithm,
1125 				 vflag);
1126     if (alg != KEYNOTE_ALGORITHM_X509)
1127       keynote_free_key(dc.dec_key, dc.dec_algorithm);
1128     keynote_free_assertion(as);
1129     if (s != NULL)
1130       free(s);
1131     return sig;
1132 }
1133 
1134 /*
1135  * ASCII-encode a key.
1136  */
1137 char *
1138 kn_encode_key(struct keynote_deckey *dc, int iencoding,
1139 	      int encoding, int keytype)
1140 {
1141     char *foo, *ptr;
1142     DSA *dsa;
1143     RSA *rsa;
1144     int i;
1145     struct keynote_binary *bn;
1146     char *s;
1147 
1148     keynote_errno = 0;
1149     if (dc == NULL || dc->dec_key == NULL)
1150     {
1151 	keynote_errno = ERROR_NOTFOUND;
1152 	return NULL;
1153     }
1154 
1155     /* DSA keys */
1156     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_DSA) &&
1157 	(iencoding == INTERNAL_ENC_ASN1) &&
1158 	((encoding == ENCODING_HEX) || (encoding == ENCODING_BASE64)))
1159     {
1160 	dsa = (DSA *) dc->dec_key;
1161 	if (keytype == KEYNOTE_PUBLIC_KEY)
1162 	  i = i2d_DSAPublicKey(dsa, NULL);
1163 	else
1164 	  i = i2d_DSAPrivateKey(dsa, NULL);
1165 
1166 	if (i <= 0)
1167 	{
1168 	    keynote_errno = ERROR_SYNTAX;
1169 	    return NULL;
1170 	}
1171 
1172  	ptr = foo = calloc(i, sizeof(char));
1173 	if (foo == NULL)
1174 	{
1175 	    keynote_errno = ERROR_MEMORY;
1176 	    return NULL;
1177 	}
1178 
1179 	if (keytype == KEYNOTE_PUBLIC_KEY)
1180 	  i2d_DSAPublicKey(dsa, (unsigned char **) &foo);
1181 	else
1182 	  i2d_DSAPrivateKey(dsa, (unsigned char **) &foo);
1183 
1184 	if (encoding == ENCODING_HEX)
1185 	{
1186 	    if (kn_encode_hex(ptr, &s, i) != 0)
1187 	    {
1188 		free(ptr);
1189 		return NULL;
1190 	    }
1191 
1192 	    free(ptr);
1193 	    return s;
1194 	}
1195 	else
1196 	  if (encoding == ENCODING_BASE64)
1197 	  {
1198 	      s = calloc(2 * i, sizeof(char));
1199 	      if (s == NULL)
1200 	      {
1201 		  free(ptr);
1202 		  keynote_errno = ERROR_MEMORY;
1203 		  return NULL;
1204 	      }
1205 
1206 	      if (kn_encode_base64(ptr, i, s, 2 * i) == -1)
1207 	      {
1208 		  free(s);
1209 		  free(ptr);
1210 		  return NULL;
1211 	      }
1212 
1213 	      free(ptr);
1214 	      return s;
1215 	  }
1216     }
1217 
1218     /* RSA keys */
1219     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_RSA) &&
1220 	(iencoding == INTERNAL_ENC_PKCS1) &&
1221 	((encoding == ENCODING_HEX) || (encoding == ENCODING_BASE64)))
1222     {
1223 	rsa = (RSA *) dc->dec_key;
1224 	if (keytype == KEYNOTE_PUBLIC_KEY)
1225 	  i = i2d_RSAPublicKey(rsa, NULL);
1226 	else
1227 	  i = i2d_RSAPrivateKey(rsa, NULL);
1228 
1229 	if (i <= 0)
1230 	{
1231 	    keynote_errno = ERROR_SYNTAX;
1232 	    return NULL;
1233 	}
1234 
1235 	ptr = foo = calloc(i, sizeof(char));
1236 	if (foo == NULL)
1237 	{
1238 	    keynote_errno = ERROR_MEMORY;
1239 	    return NULL;
1240 	}
1241 
1242 	if (keytype == KEYNOTE_PUBLIC_KEY)
1243 	  i2d_RSAPublicKey(rsa, (unsigned char **) &foo);
1244 	else
1245 	  i2d_RSAPrivateKey(rsa, (unsigned char **) &foo);
1246 
1247 	if (encoding == ENCODING_HEX)
1248 	{
1249 	    if (kn_encode_hex(ptr, &s, i) != 0)
1250 	    {
1251 		free(ptr);
1252 		return NULL;
1253 	    }
1254 
1255 	    free(ptr);
1256 	    return s;
1257 	}
1258 	else
1259 	  if (encoding == ENCODING_BASE64)
1260 	  {
1261 	      s = calloc(2 * i, sizeof(char));
1262 	      if (s == NULL)
1263 	      {
1264 		  free(ptr);
1265 		  keynote_errno = ERROR_MEMORY;
1266 		  return NULL;
1267 	      }
1268 
1269 	      if (kn_encode_base64(ptr, i, s, 2 * i) == -1)
1270 	      {
1271 		  free(s);
1272 		  free(ptr);
1273 		  return NULL;
1274 	      }
1275 
1276 	      free(ptr);
1277 	      return s;
1278 	  }
1279     }
1280 
1281     /* BINARY keys */
1282     if ((dc->dec_algorithm == KEYNOTE_ALGORITHM_BINARY) &&
1283 	(iencoding == INTERNAL_ENC_NONE) &&
1284 	((encoding == ENCODING_HEX) || (encoding == ENCODING_BASE64)))
1285     {
1286 	bn = (struct keynote_binary *) dc->dec_key;
1287 
1288 	if (encoding == ENCODING_HEX)
1289 	{
1290 	    if (kn_encode_hex(bn->bn_key, &s, bn->bn_len) != 0)
1291 	      return NULL;
1292 
1293 	    return s;
1294 	}
1295 	else
1296 	  if (encoding == ENCODING_BASE64)
1297 	  {
1298 	      s = calloc(2 * bn->bn_len, sizeof(char));
1299 	      if (s == NULL)
1300 	      {
1301 		  keynote_errno = ERROR_MEMORY;
1302 		  return NULL;
1303 	      }
1304 
1305 	      if (kn_encode_base64(bn->bn_key, bn->bn_len, s,
1306 				   2 * bn->bn_len) == -1)
1307 	      {
1308 		  free(s);
1309 		  return NULL;
1310 	      }
1311 
1312 	      return s;
1313 	  }
1314     }
1315 
1316     keynote_errno = ERROR_NOTFOUND;
1317     return NULL;
1318 }
1319