xref: /onnv-gate/usr/src/cmd/ssh/libssh/common/ssh-dss.c (revision 0:68f95e015346)
1*0Sstevel@tonic-gate /*
2*0Sstevel@tonic-gate  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
3*0Sstevel@tonic-gate  *
4*0Sstevel@tonic-gate  * Redistribution and use in source and binary forms, with or without
5*0Sstevel@tonic-gate  * modification, are permitted provided that the following conditions
6*0Sstevel@tonic-gate  * are met:
7*0Sstevel@tonic-gate  * 1. Redistributions of source code must retain the above copyright
8*0Sstevel@tonic-gate  *    notice, this list of conditions and the following disclaimer.
9*0Sstevel@tonic-gate  * 2. Redistributions in binary form must reproduce the above copyright
10*0Sstevel@tonic-gate  *    notice, this list of conditions and the following disclaimer in the
11*0Sstevel@tonic-gate  *    documentation and/or other materials provided with the distribution.
12*0Sstevel@tonic-gate  *
13*0Sstevel@tonic-gate  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14*0Sstevel@tonic-gate  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15*0Sstevel@tonic-gate  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16*0Sstevel@tonic-gate  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17*0Sstevel@tonic-gate  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18*0Sstevel@tonic-gate  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19*0Sstevel@tonic-gate  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20*0Sstevel@tonic-gate  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21*0Sstevel@tonic-gate  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22*0Sstevel@tonic-gate  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23*0Sstevel@tonic-gate  */
24*0Sstevel@tonic-gate 
25*0Sstevel@tonic-gate #include "includes.h"
26*0Sstevel@tonic-gate RCSID("$OpenBSD: ssh-dss.c,v 1.17 2002/07/04 10:41:47 markus Exp $");
27*0Sstevel@tonic-gate 
28*0Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
29*0Sstevel@tonic-gate 
30*0Sstevel@tonic-gate #include <openssl/bn.h>
31*0Sstevel@tonic-gate #include <openssl/evp.h>
32*0Sstevel@tonic-gate 
33*0Sstevel@tonic-gate #include "xmalloc.h"
34*0Sstevel@tonic-gate #include "buffer.h"
35*0Sstevel@tonic-gate #include "bufaux.h"
36*0Sstevel@tonic-gate #include "compat.h"
37*0Sstevel@tonic-gate #include "log.h"
38*0Sstevel@tonic-gate #include "key.h"
39*0Sstevel@tonic-gate #include "ssh-dss.h"
40*0Sstevel@tonic-gate 
41*0Sstevel@tonic-gate #define INTBLOB_LEN	20
42*0Sstevel@tonic-gate #define SIGBLOB_LEN	(2*INTBLOB_LEN)
43*0Sstevel@tonic-gate 
44*0Sstevel@tonic-gate int
ssh_dss_sign(Key * key,u_char ** sigp,u_int * lenp,u_char * data,u_int datalen)45*0Sstevel@tonic-gate ssh_dss_sign(Key *key, u_char **sigp, u_int *lenp,
46*0Sstevel@tonic-gate     u_char *data, u_int datalen)
47*0Sstevel@tonic-gate {
48*0Sstevel@tonic-gate 	DSA_SIG *sig;
49*0Sstevel@tonic-gate 	const EVP_MD *evp_md = EVP_sha1();
50*0Sstevel@tonic-gate 	EVP_MD_CTX md;
51*0Sstevel@tonic-gate 	u_char digest[EVP_MAX_MD_SIZE], sigblob[SIGBLOB_LEN];
52*0Sstevel@tonic-gate 	u_int rlen, slen, len, dlen;
53*0Sstevel@tonic-gate 	Buffer b;
54*0Sstevel@tonic-gate 
55*0Sstevel@tonic-gate 	if (key == NULL || key->type != KEY_DSA || key->dsa == NULL) {
56*0Sstevel@tonic-gate 		error("ssh_dss_sign: no DSA key");
57*0Sstevel@tonic-gate 		return -1;
58*0Sstevel@tonic-gate 	}
59*0Sstevel@tonic-gate 	EVP_DigestInit(&md, evp_md);
60*0Sstevel@tonic-gate 	EVP_DigestUpdate(&md, data, datalen);
61*0Sstevel@tonic-gate 	EVP_DigestFinal(&md, digest, &dlen);
62*0Sstevel@tonic-gate 
63*0Sstevel@tonic-gate 	sig = DSA_do_sign(digest, dlen, key->dsa);
64*0Sstevel@tonic-gate 	memset(digest, 'd', sizeof(digest));
65*0Sstevel@tonic-gate 
66*0Sstevel@tonic-gate 	if (sig == NULL) {
67*0Sstevel@tonic-gate 		error("ssh_dss_sign: sign failed");
68*0Sstevel@tonic-gate 		return -1;
69*0Sstevel@tonic-gate 	}
70*0Sstevel@tonic-gate 
71*0Sstevel@tonic-gate 	rlen = BN_num_bytes(sig->r);
72*0Sstevel@tonic-gate 	slen = BN_num_bytes(sig->s);
73*0Sstevel@tonic-gate 	if (rlen > INTBLOB_LEN || slen > INTBLOB_LEN) {
74*0Sstevel@tonic-gate 		error("bad sig size %u %u", rlen, slen);
75*0Sstevel@tonic-gate 		DSA_SIG_free(sig);
76*0Sstevel@tonic-gate 		return -1;
77*0Sstevel@tonic-gate 	}
78*0Sstevel@tonic-gate 	memset(sigblob, 0, SIGBLOB_LEN);
79*0Sstevel@tonic-gate 	BN_bn2bin(sig->r, sigblob+ SIGBLOB_LEN - INTBLOB_LEN - rlen);
80*0Sstevel@tonic-gate 	BN_bn2bin(sig->s, sigblob+ SIGBLOB_LEN - slen);
81*0Sstevel@tonic-gate 	DSA_SIG_free(sig);
82*0Sstevel@tonic-gate 
83*0Sstevel@tonic-gate 	if (datafellows & SSH_BUG_SIGBLOB) {
84*0Sstevel@tonic-gate 		if (lenp != NULL)
85*0Sstevel@tonic-gate 			*lenp = SIGBLOB_LEN;
86*0Sstevel@tonic-gate 		if (sigp != NULL) {
87*0Sstevel@tonic-gate 			*sigp = xmalloc(SIGBLOB_LEN);
88*0Sstevel@tonic-gate 			memcpy(*sigp, sigblob, SIGBLOB_LEN);
89*0Sstevel@tonic-gate 		}
90*0Sstevel@tonic-gate 	} else {
91*0Sstevel@tonic-gate 		/* ietf-drafts */
92*0Sstevel@tonic-gate 		buffer_init(&b);
93*0Sstevel@tonic-gate 		buffer_put_cstring(&b, "ssh-dss");
94*0Sstevel@tonic-gate 		buffer_put_string(&b, sigblob, SIGBLOB_LEN);
95*0Sstevel@tonic-gate 		len = buffer_len(&b);
96*0Sstevel@tonic-gate 		if (lenp != NULL)
97*0Sstevel@tonic-gate 			*lenp = len;
98*0Sstevel@tonic-gate 		if (sigp != NULL) {
99*0Sstevel@tonic-gate 			*sigp = xmalloc(len);
100*0Sstevel@tonic-gate 			memcpy(*sigp, buffer_ptr(&b), len);
101*0Sstevel@tonic-gate 		}
102*0Sstevel@tonic-gate 		buffer_free(&b);
103*0Sstevel@tonic-gate 	}
104*0Sstevel@tonic-gate 	return 0;
105*0Sstevel@tonic-gate }
106*0Sstevel@tonic-gate int
ssh_dss_verify(Key * key,u_char * signature,u_int signaturelen,u_char * data,u_int datalen)107*0Sstevel@tonic-gate ssh_dss_verify(Key *key, u_char *signature, u_int signaturelen,
108*0Sstevel@tonic-gate     u_char *data, u_int datalen)
109*0Sstevel@tonic-gate {
110*0Sstevel@tonic-gate 	DSA_SIG *sig;
111*0Sstevel@tonic-gate 	const EVP_MD *evp_md = EVP_sha1();
112*0Sstevel@tonic-gate 	EVP_MD_CTX md;
113*0Sstevel@tonic-gate 	u_char digest[EVP_MAX_MD_SIZE], *sigblob;
114*0Sstevel@tonic-gate 	u_int len, dlen;
115*0Sstevel@tonic-gate 	int rlen, ret;
116*0Sstevel@tonic-gate 	Buffer b;
117*0Sstevel@tonic-gate 
118*0Sstevel@tonic-gate 	if (key == NULL || key->type != KEY_DSA || key->dsa == NULL) {
119*0Sstevel@tonic-gate 		error("ssh_dss_verify: no DSA key");
120*0Sstevel@tonic-gate 		return -1;
121*0Sstevel@tonic-gate 	}
122*0Sstevel@tonic-gate 
123*0Sstevel@tonic-gate 	/* fetch signature */
124*0Sstevel@tonic-gate 	if (datafellows & SSH_BUG_SIGBLOB) {
125*0Sstevel@tonic-gate 		sigblob = signature;
126*0Sstevel@tonic-gate 		len = signaturelen;
127*0Sstevel@tonic-gate 	} else {
128*0Sstevel@tonic-gate 		/* ietf-drafts */
129*0Sstevel@tonic-gate 		char *ktype;
130*0Sstevel@tonic-gate 		buffer_init(&b);
131*0Sstevel@tonic-gate 		buffer_append(&b, signature, signaturelen);
132*0Sstevel@tonic-gate 		ktype = buffer_get_string(&b, NULL);
133*0Sstevel@tonic-gate 		if (strcmp("ssh-dss", ktype) != 0) {
134*0Sstevel@tonic-gate 			error("ssh_dss_verify: cannot handle type %s", ktype);
135*0Sstevel@tonic-gate 			buffer_free(&b);
136*0Sstevel@tonic-gate 			xfree(ktype);
137*0Sstevel@tonic-gate 			return -1;
138*0Sstevel@tonic-gate 		}
139*0Sstevel@tonic-gate 		xfree(ktype);
140*0Sstevel@tonic-gate 		sigblob = buffer_get_string(&b, &len);
141*0Sstevel@tonic-gate 		rlen = buffer_len(&b);
142*0Sstevel@tonic-gate 		buffer_free(&b);
143*0Sstevel@tonic-gate 		if (rlen != 0) {
144*0Sstevel@tonic-gate 			error("ssh_dss_verify: "
145*0Sstevel@tonic-gate 			    "remaining bytes in signature %d", rlen);
146*0Sstevel@tonic-gate 			xfree(sigblob);
147*0Sstevel@tonic-gate 			return -1;
148*0Sstevel@tonic-gate 		}
149*0Sstevel@tonic-gate 	}
150*0Sstevel@tonic-gate 
151*0Sstevel@tonic-gate 	if (len != SIGBLOB_LEN) {
152*0Sstevel@tonic-gate 		fatal("bad sigbloblen %u != SIGBLOB_LEN", len);
153*0Sstevel@tonic-gate 	}
154*0Sstevel@tonic-gate 
155*0Sstevel@tonic-gate 	/* parse signature */
156*0Sstevel@tonic-gate 	if ((sig = DSA_SIG_new()) == NULL)
157*0Sstevel@tonic-gate 		fatal("ssh_dss_verify: DSA_SIG_new failed");
158*0Sstevel@tonic-gate 	if ((sig->r = BN_new()) == NULL)
159*0Sstevel@tonic-gate 		fatal("ssh_dss_verify: BN_new failed");
160*0Sstevel@tonic-gate 	if ((sig->s = BN_new()) == NULL)
161*0Sstevel@tonic-gate 		fatal("ssh_dss_verify: BN_new failed");
162*0Sstevel@tonic-gate 	BN_bin2bn(sigblob, INTBLOB_LEN, sig->r);
163*0Sstevel@tonic-gate 	BN_bin2bn(sigblob+ INTBLOB_LEN, INTBLOB_LEN, sig->s);
164*0Sstevel@tonic-gate 
165*0Sstevel@tonic-gate 	if (!(datafellows & SSH_BUG_SIGBLOB)) {
166*0Sstevel@tonic-gate 		memset(sigblob, 0, len);
167*0Sstevel@tonic-gate 		xfree(sigblob);
168*0Sstevel@tonic-gate 	}
169*0Sstevel@tonic-gate 
170*0Sstevel@tonic-gate 	/* sha1 the data */
171*0Sstevel@tonic-gate 	EVP_DigestInit(&md, evp_md);
172*0Sstevel@tonic-gate 	EVP_DigestUpdate(&md, data, datalen);
173*0Sstevel@tonic-gate 	EVP_DigestFinal(&md, digest, &dlen);
174*0Sstevel@tonic-gate 
175*0Sstevel@tonic-gate 	ret = DSA_do_verify(digest, dlen, sig, key->dsa);
176*0Sstevel@tonic-gate 	memset(digest, 'd', sizeof(digest));
177*0Sstevel@tonic-gate 
178*0Sstevel@tonic-gate 	DSA_SIG_free(sig);
179*0Sstevel@tonic-gate 
180*0Sstevel@tonic-gate 	debug("ssh_dss_verify: signature %s",
181*0Sstevel@tonic-gate 	    ret == 1 ? "correct" : ret == 0 ? "incorrect" : "error");
182*0Sstevel@tonic-gate 	return ret;
183*0Sstevel@tonic-gate }
184