xref: /dflybsd-src/crypto/openssh/ssh-rsa.c (revision ba1276acd1c8c22d225b1bcf370a14c878644f44)
1*ba1276acSMatthew Dillon /* $OpenBSD: ssh-rsa.c,v 1.79 2023/03/05 05:34:09 dtucker Exp $ */
218de8d7fSPeter Avalos /*
318de8d7fSPeter Avalos  * Copyright (c) 2000, 2003 Markus Friedl <markus@openbsd.org>
418de8d7fSPeter Avalos  *
518de8d7fSPeter Avalos  * Permission to use, copy, modify, and distribute this software for any
618de8d7fSPeter Avalos  * purpose with or without fee is hereby granted, provided that the above
718de8d7fSPeter Avalos  * copyright notice and this permission notice appear in all copies.
818de8d7fSPeter Avalos  *
918de8d7fSPeter Avalos  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
1018de8d7fSPeter Avalos  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
1118de8d7fSPeter Avalos  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
1218de8d7fSPeter Avalos  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
1318de8d7fSPeter Avalos  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
1418de8d7fSPeter Avalos  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
1518de8d7fSPeter Avalos  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
1618de8d7fSPeter Avalos  */
1718de8d7fSPeter Avalos 
1818de8d7fSPeter Avalos #include "includes.h"
1918de8d7fSPeter Avalos 
20e9778795SPeter Avalos #ifdef WITH_OPENSSL
21e9778795SPeter Avalos 
2218de8d7fSPeter Avalos #include <sys/types.h>
2318de8d7fSPeter Avalos 
2418de8d7fSPeter Avalos #include <openssl/evp.h>
2518de8d7fSPeter Avalos #include <openssl/err.h>
2618de8d7fSPeter Avalos 
2718de8d7fSPeter Avalos #include <stdarg.h>
2818de8d7fSPeter Avalos #include <string.h>
2918de8d7fSPeter Avalos 
3036e94dc5SPeter Avalos #include "sshbuf.h"
3136e94dc5SPeter Avalos #include "ssherr.h"
3236e94dc5SPeter Avalos #define SSHKEY_INTERNAL
3336e94dc5SPeter Avalos #include "sshkey.h"
3436e94dc5SPeter Avalos #include "digest.h"
35664f4763Szrj #include "log.h"
36664f4763Szrj 
37664f4763Szrj #include "openbsd-compat/openssl-compat.h"
3818de8d7fSPeter Avalos 
3936e94dc5SPeter Avalos static int openssh_RSA_verify(int, u_char *, size_t, u_char *, size_t, RSA *);
4018de8d7fSPeter Avalos 
41*ba1276acSMatthew Dillon static u_int
ssh_rsa_size(const struct sshkey * key)42*ba1276acSMatthew Dillon ssh_rsa_size(const struct sshkey *key)
43*ba1276acSMatthew Dillon {
44*ba1276acSMatthew Dillon 	const BIGNUM *rsa_n;
45*ba1276acSMatthew Dillon 
46*ba1276acSMatthew Dillon 	if (key->rsa == NULL)
47*ba1276acSMatthew Dillon 		return 0;
48*ba1276acSMatthew Dillon 	RSA_get0_key(key->rsa, &rsa_n, NULL, NULL);
49*ba1276acSMatthew Dillon 	return BN_num_bits(rsa_n);
50*ba1276acSMatthew Dillon }
51*ba1276acSMatthew Dillon 
52*ba1276acSMatthew Dillon static int
ssh_rsa_alloc(struct sshkey * k)53*ba1276acSMatthew Dillon ssh_rsa_alloc(struct sshkey *k)
54*ba1276acSMatthew Dillon {
55*ba1276acSMatthew Dillon 	if ((k->rsa = RSA_new()) == NULL)
56*ba1276acSMatthew Dillon 		return SSH_ERR_ALLOC_FAIL;
57*ba1276acSMatthew Dillon 	return 0;
58*ba1276acSMatthew Dillon }
59*ba1276acSMatthew Dillon 
60*ba1276acSMatthew Dillon static void
ssh_rsa_cleanup(struct sshkey * k)61*ba1276acSMatthew Dillon ssh_rsa_cleanup(struct sshkey *k)
62*ba1276acSMatthew Dillon {
63*ba1276acSMatthew Dillon 	RSA_free(k->rsa);
64*ba1276acSMatthew Dillon 	k->rsa = NULL;
65*ba1276acSMatthew Dillon }
66*ba1276acSMatthew Dillon 
67*ba1276acSMatthew Dillon static int
ssh_rsa_equal(const struct sshkey * a,const struct sshkey * b)68*ba1276acSMatthew Dillon ssh_rsa_equal(const struct sshkey *a, const struct sshkey *b)
69*ba1276acSMatthew Dillon {
70*ba1276acSMatthew Dillon 	const BIGNUM *rsa_e_a, *rsa_n_a;
71*ba1276acSMatthew Dillon 	const BIGNUM *rsa_e_b, *rsa_n_b;
72*ba1276acSMatthew Dillon 
73*ba1276acSMatthew Dillon 	if (a->rsa == NULL || b->rsa == NULL)
74*ba1276acSMatthew Dillon 		return 0;
75*ba1276acSMatthew Dillon 	RSA_get0_key(a->rsa, &rsa_n_a, &rsa_e_a, NULL);
76*ba1276acSMatthew Dillon 	RSA_get0_key(b->rsa, &rsa_n_b, &rsa_e_b, NULL);
77*ba1276acSMatthew Dillon 	if (rsa_e_a == NULL || rsa_e_b == NULL)
78*ba1276acSMatthew Dillon 		return 0;
79*ba1276acSMatthew Dillon 	if (rsa_n_a == NULL || rsa_n_b == NULL)
80*ba1276acSMatthew Dillon 		return 0;
81*ba1276acSMatthew Dillon 	if (BN_cmp(rsa_e_a, rsa_e_b) != 0)
82*ba1276acSMatthew Dillon 		return 0;
83*ba1276acSMatthew Dillon 	if (BN_cmp(rsa_n_a, rsa_n_b) != 0)
84*ba1276acSMatthew Dillon 		return 0;
85*ba1276acSMatthew Dillon 	return 1;
86*ba1276acSMatthew Dillon }
87*ba1276acSMatthew Dillon 
88*ba1276acSMatthew Dillon static int
ssh_rsa_serialize_public(const struct sshkey * key,struct sshbuf * b,enum sshkey_serialize_rep opts)89*ba1276acSMatthew Dillon ssh_rsa_serialize_public(const struct sshkey *key, struct sshbuf *b,
90*ba1276acSMatthew Dillon     enum sshkey_serialize_rep opts)
91*ba1276acSMatthew Dillon {
92*ba1276acSMatthew Dillon 	int r;
93*ba1276acSMatthew Dillon 	const BIGNUM *rsa_n, *rsa_e;
94*ba1276acSMatthew Dillon 
95*ba1276acSMatthew Dillon 	if (key->rsa == NULL)
96*ba1276acSMatthew Dillon 		return SSH_ERR_INVALID_ARGUMENT;
97*ba1276acSMatthew Dillon 	RSA_get0_key(key->rsa, &rsa_n, &rsa_e, NULL);
98*ba1276acSMatthew Dillon 	if ((r = sshbuf_put_bignum2(b, rsa_e)) != 0 ||
99*ba1276acSMatthew Dillon 	    (r = sshbuf_put_bignum2(b, rsa_n)) != 0)
100*ba1276acSMatthew Dillon 		return r;
101*ba1276acSMatthew Dillon 
102*ba1276acSMatthew Dillon 	return 0;
103*ba1276acSMatthew Dillon }
104*ba1276acSMatthew Dillon 
105*ba1276acSMatthew Dillon static int
ssh_rsa_serialize_private(const struct sshkey * key,struct sshbuf * b,enum sshkey_serialize_rep opts)106*ba1276acSMatthew Dillon ssh_rsa_serialize_private(const struct sshkey *key, struct sshbuf *b,
107*ba1276acSMatthew Dillon     enum sshkey_serialize_rep opts)
108*ba1276acSMatthew Dillon {
109*ba1276acSMatthew Dillon 	int r;
110*ba1276acSMatthew Dillon 	const BIGNUM *rsa_n, *rsa_e, *rsa_d, *rsa_iqmp, *rsa_p, *rsa_q;
111*ba1276acSMatthew Dillon 
112*ba1276acSMatthew Dillon 	RSA_get0_key(key->rsa, &rsa_n, &rsa_e, &rsa_d);
113*ba1276acSMatthew Dillon 	RSA_get0_factors(key->rsa, &rsa_p, &rsa_q);
114*ba1276acSMatthew Dillon 	RSA_get0_crt_params(key->rsa, NULL, NULL, &rsa_iqmp);
115*ba1276acSMatthew Dillon 
116*ba1276acSMatthew Dillon 	if (!sshkey_is_cert(key)) {
117*ba1276acSMatthew Dillon 		/* Note: can't reuse ssh_rsa_serialize_public: e, n vs. n, e */
118*ba1276acSMatthew Dillon 		if ((r = sshbuf_put_bignum2(b, rsa_n)) != 0 ||
119*ba1276acSMatthew Dillon 		    (r = sshbuf_put_bignum2(b, rsa_e)) != 0)
120*ba1276acSMatthew Dillon 			return r;
121*ba1276acSMatthew Dillon 	}
122*ba1276acSMatthew Dillon 	if ((r = sshbuf_put_bignum2(b, rsa_d)) != 0 ||
123*ba1276acSMatthew Dillon 	    (r = sshbuf_put_bignum2(b, rsa_iqmp)) != 0 ||
124*ba1276acSMatthew Dillon 	    (r = sshbuf_put_bignum2(b, rsa_p)) != 0 ||
125*ba1276acSMatthew Dillon 	    (r = sshbuf_put_bignum2(b, rsa_q)) != 0)
126*ba1276acSMatthew Dillon 		return r;
127*ba1276acSMatthew Dillon 
128*ba1276acSMatthew Dillon 	return 0;
129*ba1276acSMatthew Dillon }
130*ba1276acSMatthew Dillon 
131*ba1276acSMatthew Dillon static int
ssh_rsa_generate(struct sshkey * k,int bits)132*ba1276acSMatthew Dillon ssh_rsa_generate(struct sshkey *k, int bits)
133*ba1276acSMatthew Dillon {
134*ba1276acSMatthew Dillon 	RSA *private = NULL;
135*ba1276acSMatthew Dillon 	BIGNUM *f4 = NULL;
136*ba1276acSMatthew Dillon 	int ret = SSH_ERR_INTERNAL_ERROR;
137*ba1276acSMatthew Dillon 
138*ba1276acSMatthew Dillon 	if (bits < SSH_RSA_MINIMUM_MODULUS_SIZE ||
139*ba1276acSMatthew Dillon 	    bits > SSHBUF_MAX_BIGNUM * 8)
140*ba1276acSMatthew Dillon 		return SSH_ERR_KEY_LENGTH;
141*ba1276acSMatthew Dillon 	if ((private = RSA_new()) == NULL || (f4 = BN_new()) == NULL) {
142*ba1276acSMatthew Dillon 		ret = SSH_ERR_ALLOC_FAIL;
143*ba1276acSMatthew Dillon 		goto out;
144*ba1276acSMatthew Dillon 	}
145*ba1276acSMatthew Dillon 	if (!BN_set_word(f4, RSA_F4) ||
146*ba1276acSMatthew Dillon 	    !RSA_generate_key_ex(private, bits, f4, NULL)) {
147*ba1276acSMatthew Dillon 		ret = SSH_ERR_LIBCRYPTO_ERROR;
148*ba1276acSMatthew Dillon 		goto out;
149*ba1276acSMatthew Dillon 	}
150*ba1276acSMatthew Dillon 	k->rsa = private;
151*ba1276acSMatthew Dillon 	private = NULL;
152*ba1276acSMatthew Dillon 	ret = 0;
153*ba1276acSMatthew Dillon  out:
154*ba1276acSMatthew Dillon 	RSA_free(private);
155*ba1276acSMatthew Dillon 	BN_free(f4);
156*ba1276acSMatthew Dillon 	return ret;
157*ba1276acSMatthew Dillon }
158*ba1276acSMatthew Dillon 
159*ba1276acSMatthew Dillon static int
ssh_rsa_copy_public(const struct sshkey * from,struct sshkey * to)160*ba1276acSMatthew Dillon ssh_rsa_copy_public(const struct sshkey *from, struct sshkey *to)
161*ba1276acSMatthew Dillon {
162*ba1276acSMatthew Dillon 	const BIGNUM *rsa_n, *rsa_e;
163*ba1276acSMatthew Dillon 	BIGNUM *rsa_n_dup = NULL, *rsa_e_dup = NULL;
164*ba1276acSMatthew Dillon 	int r = SSH_ERR_INTERNAL_ERROR;
165*ba1276acSMatthew Dillon 
166*ba1276acSMatthew Dillon 	RSA_get0_key(from->rsa, &rsa_n, &rsa_e, NULL);
167*ba1276acSMatthew Dillon 	if ((rsa_n_dup = BN_dup(rsa_n)) == NULL ||
168*ba1276acSMatthew Dillon 	    (rsa_e_dup = BN_dup(rsa_e)) == NULL) {
169*ba1276acSMatthew Dillon 		r = SSH_ERR_ALLOC_FAIL;
170*ba1276acSMatthew Dillon 		goto out;
171*ba1276acSMatthew Dillon 	}
172*ba1276acSMatthew Dillon 	if (!RSA_set0_key(to->rsa, rsa_n_dup, rsa_e_dup, NULL)) {
173*ba1276acSMatthew Dillon 		r = SSH_ERR_LIBCRYPTO_ERROR;
174*ba1276acSMatthew Dillon 		goto out;
175*ba1276acSMatthew Dillon 	}
176*ba1276acSMatthew Dillon 	rsa_n_dup = rsa_e_dup = NULL; /* transferred */
177*ba1276acSMatthew Dillon 	/* success */
178*ba1276acSMatthew Dillon 	r = 0;
179*ba1276acSMatthew Dillon  out:
180*ba1276acSMatthew Dillon 	BN_clear_free(rsa_n_dup);
181*ba1276acSMatthew Dillon 	BN_clear_free(rsa_e_dup);
182*ba1276acSMatthew Dillon 	return r;
183*ba1276acSMatthew Dillon }
184*ba1276acSMatthew Dillon 
185*ba1276acSMatthew Dillon static int
ssh_rsa_deserialize_public(const char * ktype,struct sshbuf * b,struct sshkey * key)186*ba1276acSMatthew Dillon ssh_rsa_deserialize_public(const char *ktype, struct sshbuf *b,
187*ba1276acSMatthew Dillon     struct sshkey *key)
188*ba1276acSMatthew Dillon {
189*ba1276acSMatthew Dillon 	int ret = SSH_ERR_INTERNAL_ERROR;
190*ba1276acSMatthew Dillon 	BIGNUM *rsa_n = NULL, *rsa_e = NULL;
191*ba1276acSMatthew Dillon 
192*ba1276acSMatthew Dillon 	if (sshbuf_get_bignum2(b, &rsa_e) != 0 ||
193*ba1276acSMatthew Dillon 	    sshbuf_get_bignum2(b, &rsa_n) != 0) {
194*ba1276acSMatthew Dillon 		ret = SSH_ERR_INVALID_FORMAT;
195*ba1276acSMatthew Dillon 		goto out;
196*ba1276acSMatthew Dillon 	}
197*ba1276acSMatthew Dillon 	if (!RSA_set0_key(key->rsa, rsa_n, rsa_e, NULL)) {
198*ba1276acSMatthew Dillon 		ret = SSH_ERR_LIBCRYPTO_ERROR;
199*ba1276acSMatthew Dillon 		goto out;
200*ba1276acSMatthew Dillon 	}
201*ba1276acSMatthew Dillon 	rsa_n = rsa_e = NULL; /* transferred */
202*ba1276acSMatthew Dillon 	if ((ret = sshkey_check_rsa_length(key, 0)) != 0)
203*ba1276acSMatthew Dillon 		goto out;
204*ba1276acSMatthew Dillon #ifdef DEBUG_PK
205*ba1276acSMatthew Dillon 	RSA_print_fp(stderr, key->rsa, 8);
206*ba1276acSMatthew Dillon #endif
207*ba1276acSMatthew Dillon 	/* success */
208*ba1276acSMatthew Dillon 	ret = 0;
209*ba1276acSMatthew Dillon  out:
210*ba1276acSMatthew Dillon 	BN_clear_free(rsa_n);
211*ba1276acSMatthew Dillon 	BN_clear_free(rsa_e);
212*ba1276acSMatthew Dillon 	return ret;
213*ba1276acSMatthew Dillon }
214*ba1276acSMatthew Dillon 
215*ba1276acSMatthew Dillon static int
ssh_rsa_deserialize_private(const char * ktype,struct sshbuf * b,struct sshkey * key)216*ba1276acSMatthew Dillon ssh_rsa_deserialize_private(const char *ktype, struct sshbuf *b,
217*ba1276acSMatthew Dillon     struct sshkey *key)
218*ba1276acSMatthew Dillon {
219*ba1276acSMatthew Dillon 	int r;
220*ba1276acSMatthew Dillon 	BIGNUM *rsa_n = NULL, *rsa_e = NULL, *rsa_d = NULL;
221*ba1276acSMatthew Dillon 	BIGNUM *rsa_iqmp = NULL, *rsa_p = NULL, *rsa_q = NULL;
222*ba1276acSMatthew Dillon 
223*ba1276acSMatthew Dillon 	/* Note: can't reuse ssh_rsa_deserialize_public: e, n vs. n, e */
224*ba1276acSMatthew Dillon 	if (!sshkey_is_cert(key)) {
225*ba1276acSMatthew Dillon 		if ((r = sshbuf_get_bignum2(b, &rsa_n)) != 0 ||
226*ba1276acSMatthew Dillon 		    (r = sshbuf_get_bignum2(b, &rsa_e)) != 0)
227*ba1276acSMatthew Dillon 			goto out;
228*ba1276acSMatthew Dillon 		if (!RSA_set0_key(key->rsa, rsa_n, rsa_e, NULL)) {
229*ba1276acSMatthew Dillon 			r = SSH_ERR_LIBCRYPTO_ERROR;
230*ba1276acSMatthew Dillon 			goto out;
231*ba1276acSMatthew Dillon 		}
232*ba1276acSMatthew Dillon 		rsa_n = rsa_e = NULL; /* transferred */
233*ba1276acSMatthew Dillon 	}
234*ba1276acSMatthew Dillon 	if ((r = sshbuf_get_bignum2(b, &rsa_d)) != 0 ||
235*ba1276acSMatthew Dillon 	    (r = sshbuf_get_bignum2(b, &rsa_iqmp)) != 0 ||
236*ba1276acSMatthew Dillon 	    (r = sshbuf_get_bignum2(b, &rsa_p)) != 0 ||
237*ba1276acSMatthew Dillon 	    (r = sshbuf_get_bignum2(b, &rsa_q)) != 0)
238*ba1276acSMatthew Dillon 		goto out;
239*ba1276acSMatthew Dillon 	if (!RSA_set0_key(key->rsa, NULL, NULL, rsa_d)) {
240*ba1276acSMatthew Dillon 		r = SSH_ERR_LIBCRYPTO_ERROR;
241*ba1276acSMatthew Dillon 		goto out;
242*ba1276acSMatthew Dillon 	}
243*ba1276acSMatthew Dillon 	rsa_d = NULL; /* transferred */
244*ba1276acSMatthew Dillon 	if (!RSA_set0_factors(key->rsa, rsa_p, rsa_q)) {
245*ba1276acSMatthew Dillon 		r = SSH_ERR_LIBCRYPTO_ERROR;
246*ba1276acSMatthew Dillon 		goto out;
247*ba1276acSMatthew Dillon 	}
248*ba1276acSMatthew Dillon 	rsa_p = rsa_q = NULL; /* transferred */
249*ba1276acSMatthew Dillon 	if ((r = sshkey_check_rsa_length(key, 0)) != 0)
250*ba1276acSMatthew Dillon 		goto out;
251*ba1276acSMatthew Dillon 	if ((r = ssh_rsa_complete_crt_parameters(key, rsa_iqmp)) != 0)
252*ba1276acSMatthew Dillon 		goto out;
253*ba1276acSMatthew Dillon 	if (RSA_blinding_on(key->rsa, NULL) != 1) {
254*ba1276acSMatthew Dillon 		r = SSH_ERR_LIBCRYPTO_ERROR;
255*ba1276acSMatthew Dillon 		goto out;
256*ba1276acSMatthew Dillon 	}
257*ba1276acSMatthew Dillon 	/* success */
258*ba1276acSMatthew Dillon 	r = 0;
259*ba1276acSMatthew Dillon  out:
260*ba1276acSMatthew Dillon 	BN_clear_free(rsa_n);
261*ba1276acSMatthew Dillon 	BN_clear_free(rsa_e);
262*ba1276acSMatthew Dillon 	BN_clear_free(rsa_d);
263*ba1276acSMatthew Dillon 	BN_clear_free(rsa_p);
264*ba1276acSMatthew Dillon 	BN_clear_free(rsa_q);
265*ba1276acSMatthew Dillon 	BN_clear_free(rsa_iqmp);
266*ba1276acSMatthew Dillon 	return r;
267*ba1276acSMatthew Dillon }
268*ba1276acSMatthew Dillon 
269e9778795SPeter Avalos static const char *
rsa_hash_alg_ident(int hash_alg)270e9778795SPeter Avalos rsa_hash_alg_ident(int hash_alg)
271e9778795SPeter Avalos {
272e9778795SPeter Avalos 	switch (hash_alg) {
273e9778795SPeter Avalos 	case SSH_DIGEST_SHA1:
274e9778795SPeter Avalos 		return "ssh-rsa";
275e9778795SPeter Avalos 	case SSH_DIGEST_SHA256:
276e9778795SPeter Avalos 		return "rsa-sha2-256";
277e9778795SPeter Avalos 	case SSH_DIGEST_SHA512:
278e9778795SPeter Avalos 		return "rsa-sha2-512";
279e9778795SPeter Avalos 	}
280e9778795SPeter Avalos 	return NULL;
281e9778795SPeter Avalos }
282e9778795SPeter Avalos 
283664f4763Szrj /*
284664f4763Szrj  * Returns the hash algorithm ID for a given algorithm identifier as used
285664f4763Szrj  * inside the signature blob,
286664f4763Szrj  */
287e9778795SPeter Avalos static int
rsa_hash_id_from_ident(const char * ident)288664f4763Szrj rsa_hash_id_from_ident(const char *ident)
289e9778795SPeter Avalos {
290664f4763Szrj 	if (strcmp(ident, "ssh-rsa") == 0)
291e9778795SPeter Avalos 		return SSH_DIGEST_SHA1;
292e9778795SPeter Avalos 	if (strcmp(ident, "rsa-sha2-256") == 0)
293e9778795SPeter Avalos 		return SSH_DIGEST_SHA256;
294e9778795SPeter Avalos 	if (strcmp(ident, "rsa-sha2-512") == 0)
295e9778795SPeter Avalos 		return SSH_DIGEST_SHA512;
296e9778795SPeter Avalos 	return -1;
297e9778795SPeter Avalos }
298e9778795SPeter Avalos 
299664f4763Szrj /*
300664f4763Szrj  * Return the hash algorithm ID for the specified key name. This includes
301664f4763Szrj  * all the cases of rsa_hash_id_from_ident() but also the certificate key
302664f4763Szrj  * types.
303664f4763Szrj  */
304664f4763Szrj static int
rsa_hash_id_from_keyname(const char * alg)305664f4763Szrj rsa_hash_id_from_keyname(const char *alg)
306664f4763Szrj {
307664f4763Szrj 	int r;
308664f4763Szrj 
309664f4763Szrj 	if ((r = rsa_hash_id_from_ident(alg)) != -1)
310664f4763Szrj 		return r;
311664f4763Szrj 	if (strcmp(alg, "ssh-rsa-cert-v01@openssh.com") == 0)
312664f4763Szrj 		return SSH_DIGEST_SHA1;
313664f4763Szrj 	if (strcmp(alg, "rsa-sha2-256-cert-v01@openssh.com") == 0)
314664f4763Szrj 		return SSH_DIGEST_SHA256;
315664f4763Szrj 	if (strcmp(alg, "rsa-sha2-512-cert-v01@openssh.com") == 0)
316664f4763Szrj 		return SSH_DIGEST_SHA512;
317664f4763Szrj 	return -1;
318664f4763Szrj }
319664f4763Szrj 
320e9778795SPeter Avalos static int
rsa_hash_alg_nid(int type)321e9778795SPeter Avalos rsa_hash_alg_nid(int type)
322e9778795SPeter Avalos {
323e9778795SPeter Avalos 	switch (type) {
324e9778795SPeter Avalos 	case SSH_DIGEST_SHA1:
325e9778795SPeter Avalos 		return NID_sha1;
326e9778795SPeter Avalos 	case SSH_DIGEST_SHA256:
327e9778795SPeter Avalos 		return NID_sha256;
328e9778795SPeter Avalos 	case SSH_DIGEST_SHA512:
329e9778795SPeter Avalos 		return NID_sha512;
330e9778795SPeter Avalos 	default:
331e9778795SPeter Avalos 		return -1;
332e9778795SPeter Avalos 	}
333e9778795SPeter Avalos }
334e9778795SPeter Avalos 
335ce74bacaSMatthew Dillon int
ssh_rsa_complete_crt_parameters(struct sshkey * key,const BIGNUM * iqmp)336664f4763Szrj ssh_rsa_complete_crt_parameters(struct sshkey *key, const BIGNUM *iqmp)
337ce74bacaSMatthew Dillon {
338664f4763Szrj 	const BIGNUM *rsa_p, *rsa_q, *rsa_d;
339664f4763Szrj 	BIGNUM *aux = NULL, *d_consttime = NULL;
340664f4763Szrj 	BIGNUM *rsa_dmq1 = NULL, *rsa_dmp1 = NULL, *rsa_iqmp = NULL;
341ce74bacaSMatthew Dillon 	BN_CTX *ctx = NULL;
342ce74bacaSMatthew Dillon 	int r;
343ce74bacaSMatthew Dillon 
344ce74bacaSMatthew Dillon 	if (key == NULL || key->rsa == NULL ||
345ce74bacaSMatthew Dillon 	    sshkey_type_plain(key->type) != KEY_RSA)
346ce74bacaSMatthew Dillon 		return SSH_ERR_INVALID_ARGUMENT;
347ce74bacaSMatthew Dillon 
348664f4763Szrj 	RSA_get0_key(key->rsa, NULL, NULL, &rsa_d);
349664f4763Szrj 	RSA_get0_factors(key->rsa, &rsa_p, &rsa_q);
350664f4763Szrj 
351ce74bacaSMatthew Dillon 	if ((ctx = BN_CTX_new()) == NULL)
352ce74bacaSMatthew Dillon 		return SSH_ERR_ALLOC_FAIL;
353664f4763Szrj 	if ((aux = BN_new()) == NULL ||
354664f4763Szrj 	    (rsa_dmq1 = BN_new()) == NULL ||
355664f4763Szrj 	    (rsa_dmp1 = BN_new()) == NULL)
356664f4763Szrj 		return SSH_ERR_ALLOC_FAIL;
357664f4763Szrj 	if ((d_consttime = BN_dup(rsa_d)) == NULL ||
358664f4763Szrj 	    (rsa_iqmp = BN_dup(iqmp)) == NULL) {
359ce74bacaSMatthew Dillon 		r = SSH_ERR_ALLOC_FAIL;
360ce74bacaSMatthew Dillon 		goto out;
361ce74bacaSMatthew Dillon 	}
362664f4763Szrj 	BN_set_flags(aux, BN_FLG_CONSTTIME);
363664f4763Szrj 	BN_set_flags(d_consttime, BN_FLG_CONSTTIME);
364ce74bacaSMatthew Dillon 
365664f4763Szrj 	if ((BN_sub(aux, rsa_q, BN_value_one()) == 0) ||
366664f4763Szrj 	    (BN_mod(rsa_dmq1, d_consttime, aux, ctx) == 0) ||
367664f4763Szrj 	    (BN_sub(aux, rsa_p, BN_value_one()) == 0) ||
368664f4763Szrj 	    (BN_mod(rsa_dmp1, d_consttime, aux, ctx) == 0)) {
369ce74bacaSMatthew Dillon 		r = SSH_ERR_LIBCRYPTO_ERROR;
370ce74bacaSMatthew Dillon 		goto out;
371ce74bacaSMatthew Dillon 	}
372664f4763Szrj 	if (!RSA_set0_crt_params(key->rsa, rsa_dmp1, rsa_dmq1, rsa_iqmp)) {
373664f4763Szrj 		r = SSH_ERR_LIBCRYPTO_ERROR;
374664f4763Szrj 		goto out;
375664f4763Szrj 	}
376664f4763Szrj 	rsa_dmp1 = rsa_dmq1 = rsa_iqmp = NULL; /* transferred */
377664f4763Szrj 	/* success */
378ce74bacaSMatthew Dillon 	r = 0;
379ce74bacaSMatthew Dillon  out:
380ce74bacaSMatthew Dillon 	BN_clear_free(aux);
381664f4763Szrj 	BN_clear_free(d_consttime);
382664f4763Szrj 	BN_clear_free(rsa_dmp1);
383664f4763Szrj 	BN_clear_free(rsa_dmq1);
384664f4763Szrj 	BN_clear_free(rsa_iqmp);
385ce74bacaSMatthew Dillon 	BN_CTX_free(ctx);
386ce74bacaSMatthew Dillon 	return r;
387ce74bacaSMatthew Dillon }
388ce74bacaSMatthew Dillon 
38918de8d7fSPeter Avalos /* RSASSA-PKCS1-v1_5 (PKCS #1 v2.0 signature) with SHA1 */
390*ba1276acSMatthew Dillon static int
ssh_rsa_sign(struct sshkey * key,u_char ** sigp,size_t * lenp,const u_char * data,size_t datalen,const char * alg,const char * sk_provider,const char * sk_pin,u_int compat)391*ba1276acSMatthew Dillon ssh_rsa_sign(struct sshkey *key,
392*ba1276acSMatthew Dillon     u_char **sigp, size_t *lenp,
393*ba1276acSMatthew Dillon     const u_char *data, size_t datalen,
394*ba1276acSMatthew Dillon     const char *alg, const char *sk_provider, const char *sk_pin, u_int compat)
39518de8d7fSPeter Avalos {
396664f4763Szrj 	const BIGNUM *rsa_n;
39736e94dc5SPeter Avalos 	u_char digest[SSH_DIGEST_MAX_LENGTH], *sig = NULL;
398664f4763Szrj 	size_t slen = 0;
399*ba1276acSMatthew Dillon 	u_int hlen, len;
400e9778795SPeter Avalos 	int nid, hash_alg, ret = SSH_ERR_INTERNAL_ERROR;
40136e94dc5SPeter Avalos 	struct sshbuf *b = NULL;
40218de8d7fSPeter Avalos 
40336e94dc5SPeter Avalos 	if (lenp != NULL)
40436e94dc5SPeter Avalos 		*lenp = 0;
40536e94dc5SPeter Avalos 	if (sigp != NULL)
40636e94dc5SPeter Avalos 		*sigp = NULL;
40718de8d7fSPeter Avalos 
408*ba1276acSMatthew Dillon 	if (alg == NULL || strlen(alg) == 0)
409e9778795SPeter Avalos 		hash_alg = SSH_DIGEST_SHA1;
410e9778795SPeter Avalos 	else
411*ba1276acSMatthew Dillon 		hash_alg = rsa_hash_id_from_keyname(alg);
412e9778795SPeter Avalos 	if (key == NULL || key->rsa == NULL || hash_alg == -1 ||
413ce74bacaSMatthew Dillon 	    sshkey_type_plain(key->type) != KEY_RSA)
41436e94dc5SPeter Avalos 		return SSH_ERR_INVALID_ARGUMENT;
415664f4763Szrj 	RSA_get0_key(key->rsa, &rsa_n, NULL, NULL);
416664f4763Szrj 	if (BN_num_bits(rsa_n) < SSH_RSA_MINIMUM_MODULUS_SIZE)
417ce74bacaSMatthew Dillon 		return SSH_ERR_KEY_LENGTH;
41818de8d7fSPeter Avalos 	slen = RSA_size(key->rsa);
41936e94dc5SPeter Avalos 	if (slen <= 0 || slen > SSHBUF_MAX_BIGNUM)
42036e94dc5SPeter Avalos 		return SSH_ERR_INVALID_ARGUMENT;
42118de8d7fSPeter Avalos 
42236e94dc5SPeter Avalos 	/* hash the data */
423e9778795SPeter Avalos 	nid = rsa_hash_alg_nid(hash_alg);
424*ba1276acSMatthew Dillon 	if ((hlen = ssh_digest_bytes(hash_alg)) == 0)
42536e94dc5SPeter Avalos 		return SSH_ERR_INTERNAL_ERROR;
42636e94dc5SPeter Avalos 	if ((ret = ssh_digest_memory(hash_alg, data, datalen,
42736e94dc5SPeter Avalos 	    digest, sizeof(digest))) != 0)
42836e94dc5SPeter Avalos 		goto out;
42918de8d7fSPeter Avalos 
43036e94dc5SPeter Avalos 	if ((sig = malloc(slen)) == NULL) {
43136e94dc5SPeter Avalos 		ret = SSH_ERR_ALLOC_FAIL;
43236e94dc5SPeter Avalos 		goto out;
43336e94dc5SPeter Avalos 	}
43418de8d7fSPeter Avalos 
435*ba1276acSMatthew Dillon 	if (RSA_sign(nid, digest, hlen, sig, &len, key->rsa) != 1) {
43636e94dc5SPeter Avalos 		ret = SSH_ERR_LIBCRYPTO_ERROR;
43736e94dc5SPeter Avalos 		goto out;
43818de8d7fSPeter Avalos 	}
43918de8d7fSPeter Avalos 	if (len < slen) {
44036e94dc5SPeter Avalos 		size_t diff = slen - len;
44118de8d7fSPeter Avalos 		memmove(sig + diff, sig, len);
44236e94dc5SPeter Avalos 		explicit_bzero(sig, diff);
44318de8d7fSPeter Avalos 	} else if (len > slen) {
44436e94dc5SPeter Avalos 		ret = SSH_ERR_INTERNAL_ERROR;
44536e94dc5SPeter Avalos 		goto out;
44618de8d7fSPeter Avalos 	}
44718de8d7fSPeter Avalos 	/* encode signature */
44836e94dc5SPeter Avalos 	if ((b = sshbuf_new()) == NULL) {
44936e94dc5SPeter Avalos 		ret = SSH_ERR_ALLOC_FAIL;
45036e94dc5SPeter Avalos 		goto out;
45136e94dc5SPeter Avalos 	}
452e9778795SPeter Avalos 	if ((ret = sshbuf_put_cstring(b, rsa_hash_alg_ident(hash_alg))) != 0 ||
45336e94dc5SPeter Avalos 	    (ret = sshbuf_put_string(b, sig, slen)) != 0)
45436e94dc5SPeter Avalos 		goto out;
45536e94dc5SPeter Avalos 	len = sshbuf_len(b);
45636e94dc5SPeter Avalos 	if (sigp != NULL) {
45736e94dc5SPeter Avalos 		if ((*sigp = malloc(len)) == NULL) {
45836e94dc5SPeter Avalos 			ret = SSH_ERR_ALLOC_FAIL;
45936e94dc5SPeter Avalos 			goto out;
46036e94dc5SPeter Avalos 		}
46136e94dc5SPeter Avalos 		memcpy(*sigp, sshbuf_ptr(b), len);
46236e94dc5SPeter Avalos 	}
46318de8d7fSPeter Avalos 	if (lenp != NULL)
46418de8d7fSPeter Avalos 		*lenp = len;
46536e94dc5SPeter Avalos 	ret = 0;
46636e94dc5SPeter Avalos  out:
46736e94dc5SPeter Avalos 	explicit_bzero(digest, sizeof(digest));
468664f4763Szrj 	freezero(sig, slen);
46936e94dc5SPeter Avalos 	sshbuf_free(b);
470e9778795SPeter Avalos 	return ret;
47118de8d7fSPeter Avalos }
47218de8d7fSPeter Avalos 
473*ba1276acSMatthew Dillon static int
ssh_rsa_verify(const struct sshkey * key,const u_char * sig,size_t siglen,const u_char * data,size_t dlen,const char * alg,u_int compat,struct sshkey_sig_details ** detailsp)47436e94dc5SPeter Avalos ssh_rsa_verify(const struct sshkey *key,
475*ba1276acSMatthew Dillon     const u_char *sig, size_t siglen,
476*ba1276acSMatthew Dillon     const u_char *data, size_t dlen, const char *alg, u_int compat,
477*ba1276acSMatthew Dillon     struct sshkey_sig_details **detailsp)
47818de8d7fSPeter Avalos {
479664f4763Szrj 	const BIGNUM *rsa_n;
480664f4763Szrj 	char *sigtype = NULL;
481664f4763Szrj 	int hash_alg, want_alg, ret = SSH_ERR_INTERNAL_ERROR;
482*ba1276acSMatthew Dillon 	size_t len = 0, diff, modlen, hlen;
48336e94dc5SPeter Avalos 	struct sshbuf *b = NULL;
48436e94dc5SPeter Avalos 	u_char digest[SSH_DIGEST_MAX_LENGTH], *osigblob, *sigblob = NULL;
48518de8d7fSPeter Avalos 
48636e94dc5SPeter Avalos 	if (key == NULL || key->rsa == NULL ||
48736e94dc5SPeter Avalos 	    sshkey_type_plain(key->type) != KEY_RSA ||
488e9778795SPeter Avalos 	    sig == NULL || siglen == 0)
48936e94dc5SPeter Avalos 		return SSH_ERR_INVALID_ARGUMENT;
490664f4763Szrj 	RSA_get0_key(key->rsa, &rsa_n, NULL, NULL);
491664f4763Szrj 	if (BN_num_bits(rsa_n) < SSH_RSA_MINIMUM_MODULUS_SIZE)
492ce74bacaSMatthew Dillon 		return SSH_ERR_KEY_LENGTH;
49336e94dc5SPeter Avalos 
494e9778795SPeter Avalos 	if ((b = sshbuf_from(sig, siglen)) == NULL)
49536e94dc5SPeter Avalos 		return SSH_ERR_ALLOC_FAIL;
496664f4763Szrj 	if (sshbuf_get_cstring(b, &sigtype, NULL) != 0) {
49736e94dc5SPeter Avalos 		ret = SSH_ERR_INVALID_FORMAT;
49836e94dc5SPeter Avalos 		goto out;
49918de8d7fSPeter Avalos 	}
500664f4763Szrj 	if ((hash_alg = rsa_hash_id_from_ident(sigtype)) == -1) {
50136e94dc5SPeter Avalos 		ret = SSH_ERR_KEY_TYPE_MISMATCH;
50236e94dc5SPeter Avalos 		goto out;
50318de8d7fSPeter Avalos 	}
504664f4763Szrj 	/*
505664f4763Szrj 	 * Allow ssh-rsa-cert-v01 certs to generate SHA2 signatures for
506664f4763Szrj 	 * legacy reasons, but otherwise the signature type should match.
507664f4763Szrj 	 */
508664f4763Szrj 	if (alg != NULL && strcmp(alg, "ssh-rsa-cert-v01@openssh.com") != 0) {
509664f4763Szrj 		if ((want_alg = rsa_hash_id_from_keyname(alg)) == -1) {
510664f4763Szrj 			ret = SSH_ERR_INVALID_ARGUMENT;
511664f4763Szrj 			goto out;
512664f4763Szrj 		}
513664f4763Szrj 		if (hash_alg != want_alg) {
514664f4763Szrj 			ret = SSH_ERR_SIGNATURE_INVALID;
515664f4763Szrj 			goto out;
516664f4763Szrj 		}
517664f4763Szrj 	}
51836e94dc5SPeter Avalos 	if (sshbuf_get_string(b, &sigblob, &len) != 0) {
51936e94dc5SPeter Avalos 		ret = SSH_ERR_INVALID_FORMAT;
52036e94dc5SPeter Avalos 		goto out;
52136e94dc5SPeter Avalos 	}
52236e94dc5SPeter Avalos 	if (sshbuf_len(b) != 0) {
52336e94dc5SPeter Avalos 		ret = SSH_ERR_UNEXPECTED_TRAILING_DATA;
52436e94dc5SPeter Avalos 		goto out;
52518de8d7fSPeter Avalos 	}
52618de8d7fSPeter Avalos 	/* RSA_verify expects a signature of RSA_size */
52718de8d7fSPeter Avalos 	modlen = RSA_size(key->rsa);
52818de8d7fSPeter Avalos 	if (len > modlen) {
52936e94dc5SPeter Avalos 		ret = SSH_ERR_KEY_BITS_MISMATCH;
53036e94dc5SPeter Avalos 		goto out;
53118de8d7fSPeter Avalos 	} else if (len < modlen) {
53236e94dc5SPeter Avalos 		diff = modlen - len;
53336e94dc5SPeter Avalos 		osigblob = sigblob;
53436e94dc5SPeter Avalos 		if ((sigblob = realloc(sigblob, modlen)) == NULL) {
53536e94dc5SPeter Avalos 			sigblob = osigblob; /* put it back for clear/free */
53636e94dc5SPeter Avalos 			ret = SSH_ERR_ALLOC_FAIL;
53736e94dc5SPeter Avalos 			goto out;
53836e94dc5SPeter Avalos 		}
53918de8d7fSPeter Avalos 		memmove(sigblob + diff, sigblob, len);
54036e94dc5SPeter Avalos 		explicit_bzero(sigblob, diff);
54118de8d7fSPeter Avalos 		len = modlen;
54218de8d7fSPeter Avalos 	}
543*ba1276acSMatthew Dillon 	if ((hlen = ssh_digest_bytes(hash_alg)) == 0) {
54436e94dc5SPeter Avalos 		ret = SSH_ERR_INTERNAL_ERROR;
54536e94dc5SPeter Avalos 		goto out;
54618de8d7fSPeter Avalos 	}
547*ba1276acSMatthew Dillon 	if ((ret = ssh_digest_memory(hash_alg, data, dlen,
54836e94dc5SPeter Avalos 	    digest, sizeof(digest))) != 0)
54936e94dc5SPeter Avalos 		goto out;
55018de8d7fSPeter Avalos 
551*ba1276acSMatthew Dillon 	ret = openssh_RSA_verify(hash_alg, digest, hlen, sigblob, len,
55236e94dc5SPeter Avalos 	    key->rsa);
55336e94dc5SPeter Avalos  out:
554664f4763Szrj 	freezero(sigblob, len);
555664f4763Szrj 	free(sigtype);
55636e94dc5SPeter Avalos 	sshbuf_free(b);
55736e94dc5SPeter Avalos 	explicit_bzero(digest, sizeof(digest));
55818de8d7fSPeter Avalos 	return ret;
55918de8d7fSPeter Avalos }
56018de8d7fSPeter Avalos 
56118de8d7fSPeter Avalos /*
56218de8d7fSPeter Avalos  * See:
56318de8d7fSPeter Avalos  * http://www.rsasecurity.com/rsalabs/pkcs/pkcs-1/
56418de8d7fSPeter Avalos  * ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-1/pkcs-1v2-1.asn
56518de8d7fSPeter Avalos  */
566e9778795SPeter Avalos 
56718de8d7fSPeter Avalos /*
56818de8d7fSPeter Avalos  * id-sha1 OBJECT IDENTIFIER ::= { iso(1) identified-organization(3)
56918de8d7fSPeter Avalos  *	oiw(14) secsig(3) algorithms(2) 26 }
57018de8d7fSPeter Avalos  */
57118de8d7fSPeter Avalos static const u_char id_sha1[] = {
57218de8d7fSPeter Avalos 	0x30, 0x21, /* type Sequence, length 0x21 (33) */
57318de8d7fSPeter Avalos 	0x30, 0x09, /* type Sequence, length 0x09 */
57418de8d7fSPeter Avalos 	0x06, 0x05, /* type OID, length 0x05 */
57518de8d7fSPeter Avalos 	0x2b, 0x0e, 0x03, 0x02, 0x1a, /* id-sha1 OID */
57618de8d7fSPeter Avalos 	0x05, 0x00, /* NULL */
57718de8d7fSPeter Avalos 	0x04, 0x14  /* Octet string, length 0x14 (20), followed by sha1 hash */
57818de8d7fSPeter Avalos };
57918de8d7fSPeter Avalos 
580e9778795SPeter Avalos /*
581e9778795SPeter Avalos  * See http://csrc.nist.gov/groups/ST/crypto_apps_infra/csor/algorithms.html
582e9778795SPeter Avalos  * id-sha256 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840)
583e9778795SPeter Avalos  *      organization(1) gov(101) csor(3) nistAlgorithm(4) hashAlgs(2)
584e9778795SPeter Avalos  *      id-sha256(1) }
585e9778795SPeter Avalos  */
586e9778795SPeter Avalos static const u_char id_sha256[] = {
587e9778795SPeter Avalos 	0x30, 0x31, /* type Sequence, length 0x31 (49) */
588e9778795SPeter Avalos 	0x30, 0x0d, /* type Sequence, length 0x0d (13) */
589e9778795SPeter Avalos 	0x06, 0x09, /* type OID, length 0x09 */
590e9778795SPeter Avalos 	0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, /* id-sha256 */
591e9778795SPeter Avalos 	0x05, 0x00, /* NULL */
592e9778795SPeter Avalos 	0x04, 0x20  /* Octet string, length 0x20 (32), followed by sha256 hash */
593e9778795SPeter Avalos };
594e9778795SPeter Avalos 
595e9778795SPeter Avalos /*
596e9778795SPeter Avalos  * See http://csrc.nist.gov/groups/ST/crypto_apps_infra/csor/algorithms.html
597e9778795SPeter Avalos  * id-sha512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840)
598e9778795SPeter Avalos  *      organization(1) gov(101) csor(3) nistAlgorithm(4) hashAlgs(2)
599e9778795SPeter Avalos  *      id-sha256(3) }
600e9778795SPeter Avalos  */
601e9778795SPeter Avalos static const u_char id_sha512[] = {
602e9778795SPeter Avalos 	0x30, 0x51, /* type Sequence, length 0x51 (81) */
603e9778795SPeter Avalos 	0x30, 0x0d, /* type Sequence, length 0x0d (13) */
604e9778795SPeter Avalos 	0x06, 0x09, /* type OID, length 0x09 */
605e9778795SPeter Avalos 	0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, /* id-sha512 */
606e9778795SPeter Avalos 	0x05, 0x00, /* NULL */
607e9778795SPeter Avalos 	0x04, 0x40  /* Octet string, length 0x40 (64), followed by sha512 hash */
608e9778795SPeter Avalos };
609e9778795SPeter Avalos 
610e9778795SPeter Avalos static int
rsa_hash_alg_oid(int hash_alg,const u_char ** oidp,size_t * oidlenp)611e9778795SPeter Avalos rsa_hash_alg_oid(int hash_alg, const u_char **oidp, size_t *oidlenp)
612e9778795SPeter Avalos {
613e9778795SPeter Avalos 	switch (hash_alg) {
614e9778795SPeter Avalos 	case SSH_DIGEST_SHA1:
615e9778795SPeter Avalos 		*oidp = id_sha1;
616e9778795SPeter Avalos 		*oidlenp = sizeof(id_sha1);
617e9778795SPeter Avalos 		break;
618e9778795SPeter Avalos 	case SSH_DIGEST_SHA256:
619e9778795SPeter Avalos 		*oidp = id_sha256;
620e9778795SPeter Avalos 		*oidlenp = sizeof(id_sha256);
621e9778795SPeter Avalos 		break;
622e9778795SPeter Avalos 	case SSH_DIGEST_SHA512:
623e9778795SPeter Avalos 		*oidp = id_sha512;
624e9778795SPeter Avalos 		*oidlenp = sizeof(id_sha512);
625e9778795SPeter Avalos 		break;
626e9778795SPeter Avalos 	default:
627e9778795SPeter Avalos 		return SSH_ERR_INVALID_ARGUMENT;
628e9778795SPeter Avalos 	}
629e9778795SPeter Avalos 	return 0;
630e9778795SPeter Avalos }
631e9778795SPeter Avalos 
63218de8d7fSPeter Avalos static int
openssh_RSA_verify(int hash_alg,u_char * hash,size_t hashlen,u_char * sigbuf,size_t siglen,RSA * rsa)63336e94dc5SPeter Avalos openssh_RSA_verify(int hash_alg, u_char *hash, size_t hashlen,
63436e94dc5SPeter Avalos     u_char *sigbuf, size_t siglen, RSA *rsa)
63518de8d7fSPeter Avalos {
636e9778795SPeter Avalos 	size_t rsasize = 0, oidlen = 0, hlen = 0;
637e9778795SPeter Avalos 	int ret, len, oidmatch, hashmatch;
63818de8d7fSPeter Avalos 	const u_char *oid = NULL;
63918de8d7fSPeter Avalos 	u_char *decrypted = NULL;
64018de8d7fSPeter Avalos 
641e9778795SPeter Avalos 	if ((ret = rsa_hash_alg_oid(hash_alg, &oid, &oidlen)) != 0)
642e9778795SPeter Avalos 		return ret;
64336e94dc5SPeter Avalos 	ret = SSH_ERR_INTERNAL_ERROR;
644e9778795SPeter Avalos 	hlen = ssh_digest_bytes(hash_alg);
64518de8d7fSPeter Avalos 	if (hashlen != hlen) {
64636e94dc5SPeter Avalos 		ret = SSH_ERR_INVALID_ARGUMENT;
64718de8d7fSPeter Avalos 		goto done;
64818de8d7fSPeter Avalos 	}
64918de8d7fSPeter Avalos 	rsasize = RSA_size(rsa);
65036e94dc5SPeter Avalos 	if (rsasize <= 0 || rsasize > SSHBUF_MAX_BIGNUM ||
65136e94dc5SPeter Avalos 	    siglen == 0 || siglen > rsasize) {
65236e94dc5SPeter Avalos 		ret = SSH_ERR_INVALID_ARGUMENT;
65318de8d7fSPeter Avalos 		goto done;
65418de8d7fSPeter Avalos 	}
65536e94dc5SPeter Avalos 	if ((decrypted = malloc(rsasize)) == NULL) {
65636e94dc5SPeter Avalos 		ret = SSH_ERR_ALLOC_FAIL;
65736e94dc5SPeter Avalos 		goto done;
65836e94dc5SPeter Avalos 	}
65918de8d7fSPeter Avalos 	if ((len = RSA_public_decrypt(siglen, sigbuf, decrypted, rsa,
66018de8d7fSPeter Avalos 	    RSA_PKCS1_PADDING)) < 0) {
66136e94dc5SPeter Avalos 		ret = SSH_ERR_LIBCRYPTO_ERROR;
66218de8d7fSPeter Avalos 		goto done;
66318de8d7fSPeter Avalos 	}
66436e94dc5SPeter Avalos 	if (len < 0 || (size_t)len != hlen + oidlen) {
66536e94dc5SPeter Avalos 		ret = SSH_ERR_INVALID_FORMAT;
66618de8d7fSPeter Avalos 		goto done;
66718de8d7fSPeter Avalos 	}
668856ea928SPeter Avalos 	oidmatch = timingsafe_bcmp(decrypted, oid, oidlen) == 0;
669856ea928SPeter Avalos 	hashmatch = timingsafe_bcmp(decrypted + oidlen, hash, hlen) == 0;
67036e94dc5SPeter Avalos 	if (!oidmatch || !hashmatch) {
67136e94dc5SPeter Avalos 		ret = SSH_ERR_SIGNATURE_INVALID;
67218de8d7fSPeter Avalos 		goto done;
67318de8d7fSPeter Avalos 	}
67436e94dc5SPeter Avalos 	ret = 0;
67518de8d7fSPeter Avalos done:
676664f4763Szrj 	freezero(decrypted, rsasize);
67718de8d7fSPeter Avalos 	return ret;
67818de8d7fSPeter Avalos }
679*ba1276acSMatthew Dillon 
680*ba1276acSMatthew Dillon static const struct sshkey_impl_funcs sshkey_rsa_funcs = {
681*ba1276acSMatthew Dillon 	/* .size = */		ssh_rsa_size,
682*ba1276acSMatthew Dillon 	/* .alloc = */		ssh_rsa_alloc,
683*ba1276acSMatthew Dillon 	/* .cleanup = */	ssh_rsa_cleanup,
684*ba1276acSMatthew Dillon 	/* .equal = */		ssh_rsa_equal,
685*ba1276acSMatthew Dillon 	/* .ssh_serialize_public = */ ssh_rsa_serialize_public,
686*ba1276acSMatthew Dillon 	/* .ssh_deserialize_public = */ ssh_rsa_deserialize_public,
687*ba1276acSMatthew Dillon 	/* .ssh_serialize_private = */ ssh_rsa_serialize_private,
688*ba1276acSMatthew Dillon 	/* .ssh_deserialize_private = */ ssh_rsa_deserialize_private,
689*ba1276acSMatthew Dillon 	/* .generate = */	ssh_rsa_generate,
690*ba1276acSMatthew Dillon 	/* .copy_public = */	ssh_rsa_copy_public,
691*ba1276acSMatthew Dillon 	/* .sign = */		ssh_rsa_sign,
692*ba1276acSMatthew Dillon 	/* .verify = */		ssh_rsa_verify,
693*ba1276acSMatthew Dillon };
694*ba1276acSMatthew Dillon 
695*ba1276acSMatthew Dillon const struct sshkey_impl sshkey_rsa_impl = {
696*ba1276acSMatthew Dillon 	/* .name = */		"ssh-rsa",
697*ba1276acSMatthew Dillon 	/* .shortname = */	"RSA",
698*ba1276acSMatthew Dillon 	/* .sigalg = */		NULL,
699*ba1276acSMatthew Dillon 	/* .type = */		KEY_RSA,
700*ba1276acSMatthew Dillon 	/* .nid = */		0,
701*ba1276acSMatthew Dillon 	/* .cert = */		0,
702*ba1276acSMatthew Dillon 	/* .sigonly = */	0,
703*ba1276acSMatthew Dillon 	/* .keybits = */	0,
704*ba1276acSMatthew Dillon 	/* .funcs = */		&sshkey_rsa_funcs,
705*ba1276acSMatthew Dillon };
706*ba1276acSMatthew Dillon 
707*ba1276acSMatthew Dillon const struct sshkey_impl sshkey_rsa_cert_impl = {
708*ba1276acSMatthew Dillon 	/* .name = */		"ssh-rsa-cert-v01@openssh.com",
709*ba1276acSMatthew Dillon 	/* .shortname = */	"RSA-CERT",
710*ba1276acSMatthew Dillon 	/* .sigalg = */		NULL,
711*ba1276acSMatthew Dillon 	/* .type = */		KEY_RSA_CERT,
712*ba1276acSMatthew Dillon 	/* .nid = */		0,
713*ba1276acSMatthew Dillon 	/* .cert = */		1,
714*ba1276acSMatthew Dillon 	/* .sigonly = */	0,
715*ba1276acSMatthew Dillon 	/* .keybits = */	0,
716*ba1276acSMatthew Dillon 	/* .funcs = */		&sshkey_rsa_funcs,
717*ba1276acSMatthew Dillon };
718*ba1276acSMatthew Dillon 
719*ba1276acSMatthew Dillon /* SHA2 signature algorithms */
720*ba1276acSMatthew Dillon 
721*ba1276acSMatthew Dillon const struct sshkey_impl sshkey_rsa_sha256_impl = {
722*ba1276acSMatthew Dillon 	/* .name = */		"rsa-sha2-256",
723*ba1276acSMatthew Dillon 	/* .shortname = */	"RSA",
724*ba1276acSMatthew Dillon 	/* .sigalg = */		NULL,
725*ba1276acSMatthew Dillon 	/* .type = */		KEY_RSA,
726*ba1276acSMatthew Dillon 	/* .nid = */		0,
727*ba1276acSMatthew Dillon 	/* .cert = */		0,
728*ba1276acSMatthew Dillon 	/* .sigonly = */	1,
729*ba1276acSMatthew Dillon 	/* .keybits = */	0,
730*ba1276acSMatthew Dillon 	/* .funcs = */		&sshkey_rsa_funcs,
731*ba1276acSMatthew Dillon };
732*ba1276acSMatthew Dillon 
733*ba1276acSMatthew Dillon const struct sshkey_impl sshkey_rsa_sha512_impl = {
734*ba1276acSMatthew Dillon 	/* .name = */		"rsa-sha2-512",
735*ba1276acSMatthew Dillon 	/* .shortname = */	"RSA",
736*ba1276acSMatthew Dillon 	/* .sigalg = */		NULL,
737*ba1276acSMatthew Dillon 	/* .type = */		KEY_RSA,
738*ba1276acSMatthew Dillon 	/* .nid = */		0,
739*ba1276acSMatthew Dillon 	/* .cert = */		0,
740*ba1276acSMatthew Dillon 	/* .sigonly = */	1,
741*ba1276acSMatthew Dillon 	/* .keybits = */	0,
742*ba1276acSMatthew Dillon 	/* .funcs = */		&sshkey_rsa_funcs,
743*ba1276acSMatthew Dillon };
744*ba1276acSMatthew Dillon 
745*ba1276acSMatthew Dillon const struct sshkey_impl sshkey_rsa_sha256_cert_impl = {
746*ba1276acSMatthew Dillon 	/* .name = */		"rsa-sha2-256-cert-v01@openssh.com",
747*ba1276acSMatthew Dillon 	/* .shortname = */	"RSA-CERT",
748*ba1276acSMatthew Dillon 	/* .sigalg = */		"rsa-sha2-256",
749*ba1276acSMatthew Dillon 	/* .type = */		KEY_RSA_CERT,
750*ba1276acSMatthew Dillon 	/* .nid = */		0,
751*ba1276acSMatthew Dillon 	/* .cert = */		1,
752*ba1276acSMatthew Dillon 	/* .sigonly = */	1,
753*ba1276acSMatthew Dillon 	/* .keybits = */	0,
754*ba1276acSMatthew Dillon 	/* .funcs = */		&sshkey_rsa_funcs,
755*ba1276acSMatthew Dillon };
756*ba1276acSMatthew Dillon 
757*ba1276acSMatthew Dillon const struct sshkey_impl sshkey_rsa_sha512_cert_impl = {
758*ba1276acSMatthew Dillon 	/* .name = */		"rsa-sha2-512-cert-v01@openssh.com",
759*ba1276acSMatthew Dillon 	/* .shortname = */	"RSA-CERT",
760*ba1276acSMatthew Dillon 	/* .sigalg = */		"rsa-sha2-512",
761*ba1276acSMatthew Dillon 	/* .type = */		KEY_RSA_CERT,
762*ba1276acSMatthew Dillon 	/* .nid = */		0,
763*ba1276acSMatthew Dillon 	/* .cert = */		1,
764*ba1276acSMatthew Dillon 	/* .sigonly = */	1,
765*ba1276acSMatthew Dillon 	/* .keybits = */	0,
766*ba1276acSMatthew Dillon 	/* .funcs = */		&sshkey_rsa_funcs,
767*ba1276acSMatthew Dillon };
768e9778795SPeter Avalos #endif /* WITH_OPENSSL */
769