xref: /openbsd-src/usr.bin/ssh/ssh-rsa.c (revision 3a3fbb3f2e2521ab7c4a56b7ff7462ebd9095ec5)
1 /*
2  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  */
24 
25 #include "includes.h"
26 RCSID("$OpenBSD: ssh-rsa.c,v 1.14 2001/12/05 10:06:12 deraadt Exp $");
27 
28 #include <openssl/evp.h>
29 #include <openssl/err.h>
30 
31 #include "xmalloc.h"
32 #include "log.h"
33 #include "buffer.h"
34 #include "bufaux.h"
35 #include "key.h"
36 #include "ssh-rsa.h"
37 #include "compat.h"
38 
39 /* RSASSA-PKCS1-v1_5 (PKCS #1 v2.0 signature) with SHA1 */
40 int
41 ssh_rsa_sign(
42     Key *key,
43     u_char **sigp, int *lenp,
44     u_char *data, int datalen)
45 {
46 	const EVP_MD *evp_md;
47 	EVP_MD_CTX md;
48 	u_char *digest, *sig, *ret;
49 	u_int slen, dlen, len;
50 	int ok, nid;
51 	Buffer b;
52 
53 	if (key == NULL || key->type != KEY_RSA || key->rsa == NULL) {
54 		error("ssh_rsa_sign: no RSA key");
55 		return -1;
56 	}
57 	if (datafellows & SSH_BUG_SIGBLOB) {
58 		error("ssh_rsa_sign: SSH_BUG_SIGBLOB not supported");
59 		return -1;
60 	}
61 	nid = (datafellows & SSH_BUG_RSASIGMD5) ? NID_md5 : NID_sha1;
62 	if ((evp_md = EVP_get_digestbynid(nid)) == NULL) {
63 		error("ssh_rsa_sign: EVP_get_digestbynid %d failed", nid);
64 		return -1;
65 	}
66 	dlen = evp_md->md_size;
67 	digest = xmalloc(dlen);
68 	EVP_DigestInit(&md, evp_md);
69 	EVP_DigestUpdate(&md, data, datalen);
70 	EVP_DigestFinal(&md, digest, NULL);
71 
72 	slen = RSA_size(key->rsa);
73 	sig = xmalloc(slen);
74 
75 	ok = RSA_sign(nid, digest, dlen, sig, &len, key->rsa);
76 	memset(digest, 'd', dlen);
77 	xfree(digest);
78 
79 	if (ok != 1) {
80 		int ecode = ERR_get_error();
81 		error("ssh_rsa_sign: RSA_sign failed: %s", ERR_error_string(ecode, NULL));
82 		xfree(sig);
83 		return -1;
84 	}
85 	if (len < slen) {
86 		int diff = slen - len;
87 		debug("slen %d > len %d", slen, len);
88 		memmove(sig + diff, sig, len);
89 		memset(sig, 0, diff);
90 	} else if (len > slen) {
91 		error("ssh_rsa_sign: slen %d slen2 %d", slen, len);
92 		xfree(sig);
93 		return -1;
94 	}
95 	/* encode signature */
96 	buffer_init(&b);
97 	buffer_put_cstring(&b, "ssh-rsa");
98 	buffer_put_string(&b, sig, slen);
99 	len = buffer_len(&b);
100 	ret = xmalloc(len);
101 	memcpy(ret, buffer_ptr(&b), len);
102 	buffer_free(&b);
103 	memset(sig, 's', slen);
104 	xfree(sig);
105 
106 	if (lenp != NULL)
107 		*lenp = len;
108 	if (sigp != NULL)
109 		*sigp = ret;
110 	return 0;
111 }
112 
113 int
114 ssh_rsa_verify(
115     Key *key,
116     u_char *signature, int signaturelen,
117     u_char *data, int datalen)
118 {
119 	Buffer b;
120 	const EVP_MD *evp_md;
121 	EVP_MD_CTX md;
122 	char *ktype;
123 	u_char *sigblob, *digest;
124 	u_int len, dlen;
125 	int rlen, ret, nid;
126 
127 	if (key == NULL || key->type != KEY_RSA || key->rsa == NULL) {
128 		error("ssh_rsa_verify: no RSA key");
129 		return -1;
130 	}
131 	if (datafellows & SSH_BUG_SIGBLOB) {
132 		error("ssh_rsa_verify: SSH_BUG_SIGBLOB not supported");
133 		return -1;
134 	}
135 	if (BN_num_bits(key->rsa->n) < 768) {
136 		error("ssh_rsa_verify: n too small: %d bits",
137 		    BN_num_bits(key->rsa->n));
138 		return -1;
139 	}
140 	buffer_init(&b);
141 	buffer_append(&b, signature, signaturelen);
142 	ktype = buffer_get_string(&b, NULL);
143 	if (strcmp("ssh-rsa", ktype) != 0) {
144 		error("ssh_rsa_verify: cannot handle type %s", ktype);
145 		buffer_free(&b);
146 		xfree(ktype);
147 		return -1;
148 	}
149 	xfree(ktype);
150 	sigblob = buffer_get_string(&b, &len);
151 	rlen = buffer_len(&b);
152 	buffer_free(&b);
153 	if (rlen != 0) {
154 		error("ssh_rsa_verify: remaining bytes in signature %d", rlen);
155 		xfree(sigblob);
156 		return -1;
157 	}
158 	nid = (datafellows & SSH_BUG_RSASIGMD5) ? NID_md5 : NID_sha1;
159 	if ((evp_md = EVP_get_digestbynid(nid)) == NULL) {
160 		error("ssh_rsa_verify: EVP_get_digestbynid %d failed", nid);
161 		xfree(sigblob);
162 		return -1;
163 	}
164 	dlen = evp_md->md_size;
165 	digest = xmalloc(dlen);
166 	EVP_DigestInit(&md, evp_md);
167 	EVP_DigestUpdate(&md, data, datalen);
168 	EVP_DigestFinal(&md, digest, NULL);
169 
170 	ret = RSA_verify(nid, digest, dlen, sigblob, len, key->rsa);
171 	memset(digest, 'd', dlen);
172 	xfree(digest);
173 	memset(sigblob, 's', len);
174 	xfree(sigblob);
175 	if (ret == 0) {
176 		int ecode = ERR_get_error();
177 		error("ssh_rsa_verify: RSA_verify failed: %s", ERR_error_string(ecode, NULL));
178 	}
179 	debug("ssh_rsa_verify: signature %scorrect", (ret==0) ? "in" : "");
180 	return ret;
181 }
182