xref: /openbsd-src/lib/libcrypto/rsa/rsa_ameth.c (revision ff0e7be1ebbcc809ea8ad2b6dafe215824da9e46)
1 /* $OpenBSD: rsa_ameth.c,v 1.29 2023/05/19 17:31:20 tb Exp $ */
2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58 
59 #include <stdio.h>
60 
61 #include <openssl/opensslconf.h>
62 
63 #include <openssl/asn1t.h>
64 #include <openssl/bn.h>
65 #include <openssl/cms.h>
66 #include <openssl/err.h>
67 #include <openssl/rsa.h>
68 #include <openssl/x509.h>
69 
70 #include "asn1_local.h"
71 #include "cryptlib.h"
72 #include "evp_local.h"
73 #include "rsa_local.h"
74 
75 #ifndef OPENSSL_NO_CMS
76 static int rsa_cms_sign(CMS_SignerInfo *si);
77 static int rsa_cms_verify(CMS_SignerInfo *si);
78 static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
79 static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
80 #endif
81 
82 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg);
83 
84 /* Set any parameters associated with pkey */
85 static int
86 rsa_param_encode(const EVP_PKEY *pkey, ASN1_STRING **pstr, int *pstrtype)
87 {
88 	const RSA *rsa = pkey->pkey.rsa;
89 
90 	*pstr = NULL;
91 
92 	/* If RSA it's just NULL type */
93 	if (pkey->ameth->pkey_id != EVP_PKEY_RSA_PSS) {
94 		*pstrtype = V_ASN1_NULL;
95 		return 1;
96 	}
97 
98 	/* If no PSS parameters we omit parameters entirely */
99 	if (rsa->pss == NULL) {
100 		*pstrtype = V_ASN1_UNDEF;
101 		return 1;
102 	}
103 
104 	/* Encode PSS parameters */
105 	if (ASN1_item_pack(rsa->pss, &RSA_PSS_PARAMS_it, pstr) == NULL)
106 		return 0;
107 
108 	*pstrtype = V_ASN1_SEQUENCE;
109 	return 1;
110 }
111 
112 /* Decode any parameters and set them in RSA structure */
113 static int
114 rsa_param_decode(RSA *rsa, const X509_ALGOR *alg)
115 {
116 	const ASN1_OBJECT *algoid;
117 	const void *algp;
118 	int algptype;
119 
120 	X509_ALGOR_get0(&algoid, &algptype, &algp, alg);
121 	if (OBJ_obj2nid(algoid) != EVP_PKEY_RSA_PSS)
122 		return 1;
123 	if (algptype == V_ASN1_UNDEF)
124 		return 1;
125 	if (algptype != V_ASN1_SEQUENCE) {
126 		RSAerror(RSA_R_INVALID_PSS_PARAMETERS);
127 		return 0;
128 	}
129 	rsa->pss = rsa_pss_decode(alg);
130 	if (rsa->pss == NULL)
131 		return 0;
132 	return 1;
133 }
134 
135 static int
136 rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
137 {
138 	unsigned char *penc = NULL;
139 	int penclen;
140 	ASN1_STRING *str;
141 	int strtype;
142 
143 	if (!rsa_param_encode(pkey, &str, &strtype))
144 		return 0;
145 	penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
146 	if (penclen <= 0)
147 		return 0;
148 	if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id),
149 	    strtype, str, penc, penclen))
150 		return 1;
151 
152 	free(penc);
153 
154 	return 0;
155 }
156 
157 static int
158 rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
159 {
160 	const unsigned char *p;
161 	int pklen;
162 	X509_ALGOR *alg;
163 	RSA *rsa = NULL;
164 
165 	if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, &alg, pubkey))
166 		return 0;
167 	if ((rsa = d2i_RSAPublicKey(NULL, &p, pklen)) == NULL) {
168 		RSAerror(ERR_R_RSA_LIB);
169 		return 0;
170 	}
171 	if (!rsa_param_decode(rsa, alg)) {
172 		RSA_free(rsa);
173 		return 0;
174 	}
175 	if (!EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa)) {
176 		RSA_free(rsa);
177 		return 0;
178 	}
179 	return 1;
180 }
181 
182 static int
183 rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
184 {
185 	if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0 ||
186 	    BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
187 		return 0;
188 
189 	return 1;
190 }
191 
192 static int
193 old_rsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
194 {
195 	RSA *rsa;
196 
197 	if ((rsa = d2i_RSAPrivateKey(NULL, pder, derlen)) == NULL) {
198 		RSAerror(ERR_R_RSA_LIB);
199 		return 0;
200 	}
201 	EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
202 	return 1;
203 }
204 
205 static int
206 old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
207 {
208 	return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
209 }
210 
211 static int
212 rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
213 {
214 	unsigned char *rk = NULL;
215 	int rklen;
216 	ASN1_STRING *str;
217 	int strtype;
218 
219 	if (!rsa_param_encode(pkey, &str, &strtype))
220 		return 0;
221 
222 	rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
223 	if (rklen <= 0) {
224 		RSAerror(ERR_R_MALLOC_FAILURE);
225 		ASN1_STRING_free(str);
226 		return 0;
227 	}
228 
229 	if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0,
230 	    strtype, str, rk, rklen)) {
231 		RSAerror(ERR_R_MALLOC_FAILURE);
232 		ASN1_STRING_free(str);
233 		return 0;
234 	}
235 
236 	return 1;
237 }
238 
239 static int
240 rsa_priv_decode(EVP_PKEY *pkey, const PKCS8_PRIV_KEY_INFO *p8)
241 {
242 	const unsigned char *p;
243 	RSA *rsa;
244 	int pklen;
245 	const X509_ALGOR *alg;
246 
247 	if (!PKCS8_pkey_get0(NULL, &p, &pklen, &alg, p8))
248 		return 0;
249 	rsa = d2i_RSAPrivateKey(NULL, &p, pklen);
250 	if (rsa == NULL) {
251 		RSAerror(ERR_R_RSA_LIB);
252 		return 0;
253 	}
254 	if (!rsa_param_decode(rsa, alg)) {
255 		RSA_free(rsa);
256 		return 0;
257 	}
258 	EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
259 
260 	return 1;
261 }
262 
263 static int
264 int_rsa_size(const EVP_PKEY *pkey)
265 {
266 	return RSA_size(pkey->pkey.rsa);
267 }
268 
269 static int
270 rsa_bits(const EVP_PKEY *pkey)
271 {
272 	return BN_num_bits(pkey->pkey.rsa->n);
273 }
274 
275 static int
276 rsa_security_bits(const EVP_PKEY *pkey)
277 {
278 	return RSA_security_bits(pkey->pkey.rsa);
279 }
280 
281 static void
282 int_rsa_free(EVP_PKEY *pkey)
283 {
284 	RSA_free(pkey->pkey.rsa);
285 }
286 
287 static X509_ALGOR *
288 rsa_mgf1_decode(X509_ALGOR *alg)
289 {
290 	if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
291 		return NULL;
292 
293 	return ASN1_TYPE_unpack_sequence(&X509_ALGOR_it, alg->parameter);
294 }
295 
296 static RSA_PSS_PARAMS *
297 rsa_pss_decode(const X509_ALGOR *alg)
298 {
299 	RSA_PSS_PARAMS *pss;
300 
301 	pss = ASN1_TYPE_unpack_sequence(&RSA_PSS_PARAMS_it, alg->parameter);
302 	if (pss == NULL)
303 		return NULL;
304 
305 	if (pss->maskGenAlgorithm != NULL) {
306 		pss->maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
307 		if (pss->maskHash == NULL) {
308 			RSA_PSS_PARAMS_free(pss);
309 			return NULL;
310 		}
311 	}
312 
313 	return pss;
314 }
315 
316 static int
317 rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss, int indent)
318 {
319 	int rv = 0;
320 	X509_ALGOR *maskHash = NULL;
321 
322 	if (!BIO_indent(bp, indent, 128))
323 		goto err;
324 	if (pss_key) {
325 		if (pss == NULL) {
326 			if (BIO_puts(bp, "No PSS parameter restrictions\n") <= 0)
327 				return 0;
328 			return 1;
329 		} else {
330 			if (BIO_puts(bp, "PSS parameter restrictions:") <= 0)
331 				return 0;
332 		}
333 	} else if (pss == NULL) {
334 		if (BIO_puts(bp,"(INVALID PSS PARAMETERS)\n") <= 0)
335 			return 0;
336 		return 1;
337 	}
338 	if (BIO_puts(bp, "\n") <= 0)
339 		goto err;
340 	if (pss_key)
341 		indent += 2;
342 	if (!BIO_indent(bp, indent, 128))
343 		goto err;
344 	if (BIO_puts(bp, "Hash Algorithm: ") <= 0)
345 		goto err;
346 
347 	if (pss->hashAlgorithm) {
348 		if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0)
349 			goto err;
350 	} else if (BIO_puts(bp, "sha1 (default)") <= 0) {
351 		goto err;
352 	}
353 
354 	if (BIO_puts(bp, "\n") <= 0)
355 		goto err;
356 
357 	if (!BIO_indent(bp, indent, 128))
358 		goto err;
359 
360 	if (BIO_puts(bp, "Mask Algorithm: ") <= 0)
361 		goto err;
362 	if (pss->maskGenAlgorithm) {
363 		if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0)
364 			goto err;
365 		if (BIO_puts(bp, " with ") <= 0)
366 			goto err;
367 		maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
368 		if (maskHash != NULL) {
369 			if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
370 				goto err;
371 		} else if (BIO_puts(bp, "INVALID") <= 0) {
372 			goto err;
373 		}
374 	} else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0) {
375 		goto err;
376 	}
377 	BIO_puts(bp, "\n");
378 
379 	if (!BIO_indent(bp, indent, 128))
380 		goto err;
381 	if (BIO_printf(bp, "%s Salt Length: 0x", pss_key ? "Minimum" : "") <= 0)
382 		goto err;
383 	if (pss->saltLength) {
384 		if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0)
385 			goto err;
386 	} else if (BIO_puts(bp, "14 (default)") <= 0) {
387 		goto err;
388 	}
389 	BIO_puts(bp, "\n");
390 
391 	if (!BIO_indent(bp, indent, 128))
392 		goto err;
393 	if (BIO_puts(bp, "Trailer Field: 0x") <= 0)
394 		goto err;
395 	if (pss->trailerField) {
396 		if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0)
397 			goto err;
398 	} else if (BIO_puts(bp, "BC (default)") <= 0) {
399 		goto err;
400 	}
401 	BIO_puts(bp, "\n");
402 
403 	rv = 1;
404 
405  err:
406 	X509_ALGOR_free(maskHash);
407 	return rv;
408 
409 }
410 
411 static void
412 update_buflen(const BIGNUM *b, size_t *pbuflen)
413 {
414 	size_t i;
415 
416 	if (!b)
417 		return;
418 	if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
419 		*pbuflen = i;
420 }
421 
422 static int
423 pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
424 {
425 	const RSA *x = pkey->pkey.rsa;
426 	unsigned char *m = NULL;
427 	char *str;
428 	const char *s;
429 	int ret = 0, mod_len = 0;
430 	size_t buf_len = 0;
431 
432 	update_buflen(x->n, &buf_len);
433 	update_buflen(x->e, &buf_len);
434 
435 	if (priv) {
436 		update_buflen(x->d, &buf_len);
437 		update_buflen(x->p, &buf_len);
438 		update_buflen(x->q, &buf_len);
439 		update_buflen(x->dmp1, &buf_len);
440 		update_buflen(x->dmq1, &buf_len);
441 		update_buflen(x->iqmp, &buf_len);
442 	}
443 
444 	m = malloc(buf_len + 10);
445 	if (m == NULL) {
446 		RSAerror(ERR_R_MALLOC_FAILURE);
447 		goto err;
448 	}
449 
450 	if (x->n != NULL)
451 		mod_len = BN_num_bits(x->n);
452 
453 	if (!BIO_indent(bp, off, 128))
454 		goto err;
455 
456 	if (BIO_printf(bp, "%s ", pkey_is_pss(pkey) ?  "RSA-PSS" : "RSA") <= 0)
457 		goto err;
458 
459 	if (priv && x->d != NULL) {
460 		if (BIO_printf(bp, "Private-Key: (%d bit)\n", mod_len) <= 0)
461 			goto err;
462 		str = "modulus:";
463 		s = "publicExponent:";
464 	} else {
465 		if (BIO_printf(bp, "Public-Key: (%d bit)\n", mod_len) <= 0)
466 			goto err;
467 		str = "Modulus:";
468 		s = "Exponent:";
469 	}
470 	if (!ASN1_bn_print(bp, str, x->n, m, off))
471 		goto err;
472 	if (!ASN1_bn_print(bp, s, x->e, m, off))
473 		goto err;
474 	if (priv) {
475 		if (!ASN1_bn_print(bp, "privateExponent:", x->d, m, off))
476 			goto err;
477 		if (!ASN1_bn_print(bp, "prime1:", x->p, m, off))
478 			goto err;
479 		if (!ASN1_bn_print(bp, "prime2:", x->q, m, off))
480 			goto err;
481 		if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, m, off))
482 			goto err;
483 		if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, m, off))
484 			goto err;
485 		if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, m, off))
486 			goto err;
487 	}
488 	if (pkey_is_pss(pkey) && !rsa_pss_param_print(bp, 1, x->pss, off))
489 		goto err;
490 	ret = 1;
491  err:
492 	free(m);
493 	return ret;
494 }
495 
496 static int
497 rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent, ASN1_PCTX *ctx)
498 {
499 	return pkey_rsa_print(bp, pkey, indent, 0);
500 }
501 
502 static int
503 rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent, ASN1_PCTX *ctx)
504 {
505 	return pkey_rsa_print(bp, pkey, indent, 1);
506 }
507 
508 static int
509 rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg, const ASN1_STRING *sig,
510     int indent, ASN1_PCTX *pctx)
511 {
512 	if (OBJ_obj2nid(sigalg->algorithm) == EVP_PKEY_RSA_PSS) {
513 		int rv;
514 		RSA_PSS_PARAMS *pss = rsa_pss_decode(sigalg);
515 
516 		rv = rsa_pss_param_print(bp, 0, pss, indent);
517 		RSA_PSS_PARAMS_free(pss);
518 		if (!rv)
519 			return 0;
520 	} else if (!sig && BIO_puts(bp, "\n") <= 0) {
521 		return 0;
522 	}
523 	if (sig)
524 		return X509_signature_dump(bp, sig, indent);
525 	return 1;
526 }
527 
528 static int
529 rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
530 {
531 	X509_ALGOR *alg = NULL;
532 	const EVP_MD *md;
533 	const EVP_MD *mgf1md;
534 	int min_saltlen;
535 
536 	switch (op) {
537 	case ASN1_PKEY_CTRL_PKCS7_SIGN:
538 		if (arg1 == 0)
539 			PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
540 		break;
541 
542 	case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
543 		if (pkey_is_pss(pkey))
544 			return -2;
545 		if (arg1 == 0)
546 			PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
547 		break;
548 #ifndef OPENSSL_NO_CMS
549 	case ASN1_PKEY_CTRL_CMS_SIGN:
550 		if (arg1 == 0)
551 			return rsa_cms_sign(arg2);
552 		else if (arg1 == 1)
553 			return rsa_cms_verify(arg2);
554 		break;
555 
556 	case ASN1_PKEY_CTRL_CMS_ENVELOPE:
557 		if (pkey_is_pss(pkey))
558 			return -2;
559 		if (arg1 == 0)
560 			return rsa_cms_encrypt(arg2);
561 		else if (arg1 == 1)
562 			return rsa_cms_decrypt(arg2);
563 		break;
564 
565 	case ASN1_PKEY_CTRL_CMS_RI_TYPE:
566 		if (pkey_is_pss(pkey))
567 			return -2;
568 		*(int *)arg2 = CMS_RECIPINFO_TRANS;
569 		return 1;
570 #endif
571 
572 	case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
573 		if (pkey->pkey.rsa->pss != NULL) {
574 			if (!rsa_pss_get_param(pkey->pkey.rsa->pss, &md, &mgf1md,
575 			    &min_saltlen)) {
576 				RSAerror(ERR_R_INTERNAL_ERROR);
577 				return 0;
578 			}
579 			*(int *)arg2 = EVP_MD_type(md);
580 			/* Return of 2 indicates this MD is mandatory */
581 			return 2;
582 		}
583 		*(int *)arg2 = NID_sha256;
584 		return 1;
585 
586 	default:
587 		return -2;
588 	}
589 
590 	if (alg)
591 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
592 		    V_ASN1_NULL, 0);
593 
594 	return 1;
595 }
596 
597 /* Allocate and set algorithm ID from EVP_MD, defaults to SHA1. */
598 static int
599 rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
600 {
601 	if (md == NULL || EVP_MD_type(md) == NID_sha1)
602 		return 1;
603 	*palg = X509_ALGOR_new();
604 	if (*palg == NULL)
605 		return 0;
606 	X509_ALGOR_set_md(*palg, md);
607 	return 1;
608 }
609 
610 /* Allocate and set MGF1 algorithm ID from EVP_MD. */
611 static int
612 rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
613 {
614 	X509_ALGOR *algtmp = NULL;
615 	ASN1_STRING *stmp = NULL;
616 
617 	*palg = NULL;
618 	if (mgf1md == NULL || EVP_MD_type(mgf1md) == NID_sha1)
619 		return 1;
620 	/* need to embed algorithm ID inside another */
621 	if (!rsa_md_to_algor(&algtmp, mgf1md))
622 		goto err;
623 	if (ASN1_item_pack(algtmp, &X509_ALGOR_it, &stmp) == NULL)
624 		 goto err;
625 	*palg = X509_ALGOR_new();
626 	if (*palg == NULL)
627 		goto err;
628 	X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
629 	stmp = NULL;
630  err:
631 	ASN1_STRING_free(stmp);
632 	X509_ALGOR_free(algtmp);
633 	if (*palg)
634 		return 1;
635 	return 0;
636 }
637 
638 /* Convert algorithm ID to EVP_MD, defaults to SHA1. */
639 static const EVP_MD *
640 rsa_algor_to_md(X509_ALGOR *alg)
641 {
642 	const EVP_MD *md;
643 
644 	if (!alg)
645 		return EVP_sha1();
646 	md = EVP_get_digestbyobj(alg->algorithm);
647 	if (md == NULL)
648 		RSAerror(RSA_R_UNKNOWN_DIGEST);
649 	return md;
650 }
651 
652 /*
653  * Convert EVP_PKEY_CTX in PSS mode into corresponding algorithm parameter,
654  * suitable for setting an AlgorithmIdentifier.
655  */
656 static RSA_PSS_PARAMS *
657 rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
658 {
659 	const EVP_MD *sigmd, *mgf1md;
660 	EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
661 	int saltlen;
662 
663 	if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
664 		return NULL;
665 	if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
666 		return NULL;
667 	if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
668 		return NULL;
669 	if (saltlen == -1) {
670 		saltlen = EVP_MD_size(sigmd);
671 	} else if (saltlen == -2 || saltlen == -3) {
672 		saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
673 		if ((EVP_PKEY_bits(pk) & 0x7) == 1)
674 			saltlen--;
675 		if (saltlen < 0)
676 			return NULL;
677 	}
678 
679 	return rsa_pss_params_create(sigmd, mgf1md, saltlen);
680 }
681 
682 RSA_PSS_PARAMS *
683 rsa_pss_params_create(const EVP_MD *sigmd, const EVP_MD *mgf1md, int saltlen)
684 {
685 	RSA_PSS_PARAMS *pss = RSA_PSS_PARAMS_new();
686 
687 	if (pss == NULL)
688 		goto err;
689 	if (saltlen != 20) {
690 		pss->saltLength = ASN1_INTEGER_new();
691 		if (pss->saltLength == NULL)
692 			goto err;
693 		if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
694 			goto err;
695 	}
696 	if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
697 		goto err;
698 	if (mgf1md == NULL)
699 		mgf1md = sigmd;
700 	if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
701 		goto err;
702 	if (!rsa_md_to_algor(&pss->maskHash, mgf1md))
703 		goto err;
704 	return pss;
705  err:
706 	RSA_PSS_PARAMS_free(pss);
707 	return NULL;
708 }
709 
710 static ASN1_STRING *
711 rsa_ctx_to_pss_string(EVP_PKEY_CTX *pkctx)
712 {
713 	RSA_PSS_PARAMS *pss = rsa_ctx_to_pss(pkctx);
714 	ASN1_STRING *os;
715 
716 	if (pss == NULL)
717 		return NULL;
718 
719 	os = ASN1_item_pack(pss, &RSA_PSS_PARAMS_it, NULL);
720 	RSA_PSS_PARAMS_free(pss);
721 	return os;
722 }
723 
724 /*
725  * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL
726  * then the EVP_MD_CTX is setup and initialised. If it is NULL parameters are
727  * passed to pkctx instead.
728  */
729 
730 static int
731 rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
732     X509_ALGOR *sigalg, EVP_PKEY *pkey)
733 {
734 	int rv = -1;
735 	int saltlen;
736 	const EVP_MD *mgf1md = NULL, *md = NULL;
737 	RSA_PSS_PARAMS *pss;
738 
739 	/* Sanity check: make sure it is PSS */
740 	if (OBJ_obj2nid(sigalg->algorithm) != EVP_PKEY_RSA_PSS) {
741 		RSAerror(RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
742 		return -1;
743 	}
744 	/* Decode PSS parameters */
745 	pss = rsa_pss_decode(sigalg);
746 
747 	if (!rsa_pss_get_param(pss, &md, &mgf1md, &saltlen)) {
748 		RSAerror(RSA_R_INVALID_PSS_PARAMETERS);
749 		goto err;
750 	}
751 
752 	/* We have all parameters now set up context */
753 	if (pkey) {
754 		if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
755 			goto err;
756 	} else {
757 		const EVP_MD *checkmd;
758 		if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0)
759 			goto err;
760 		if (EVP_MD_type(md) != EVP_MD_type(checkmd)) {
761 			RSAerror(RSA_R_DIGEST_DOES_NOT_MATCH);
762 			goto err;
763 		}
764 	}
765 
766 	if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0)
767 		goto err;
768 
769 	if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0)
770 		goto err;
771 
772 	if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
773 		goto err;
774 	/* Carry on */
775 	rv = 1;
776 
777  err:
778 	RSA_PSS_PARAMS_free(pss);
779 	return rv;
780 }
781 
782 int
783 rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,
784     const EVP_MD **pmgf1md, int *psaltlen)
785 {
786 	if (pss == NULL)
787 		return 0;
788 	*pmd = rsa_algor_to_md(pss->hashAlgorithm);
789 	if (*pmd == NULL)
790 		return 0;
791 	*pmgf1md = rsa_algor_to_md(pss->maskHash);
792 	if (*pmgf1md == NULL)
793 		return 0;
794 	if (pss->saltLength) {
795 		*psaltlen = ASN1_INTEGER_get(pss->saltLength);
796 		if (*psaltlen < 0) {
797 			RSAerror(RSA_R_INVALID_SALT_LENGTH);
798 			return 0;
799 		}
800 	} else {
801 		*psaltlen = 20;
802 	}
803 
804 	/*
805 	 * low-level routines support only trailer field 0xbc (value 1) and
806 	 * PKCS#1 says we should reject any other value anyway.
807 	 */
808 	if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
809 		RSAerror(RSA_R_INVALID_TRAILER);
810 		return 0;
811 	}
812 
813 	return 1;
814 }
815 
816 #ifndef OPENSSL_NO_CMS
817 static int
818 rsa_cms_verify(CMS_SignerInfo *si)
819 {
820 	int nid, nid2;
821 	X509_ALGOR *alg;
822 	EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
823 
824 	CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
825 	nid = OBJ_obj2nid(alg->algorithm);
826 	if (nid == EVP_PKEY_RSA_PSS)
827 		return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
828 	/* Only PSS allowed for PSS keys */
829 	if (pkey_ctx_is_pss(pkctx)) {
830 		RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
831 		return 0;
832 	}
833 	if (nid == NID_rsaEncryption)
834 		return 1;
835 	/* Workaround for some implementation that use a signature OID */
836 	if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
837 		if (nid2 == NID_rsaEncryption)
838 			return 1;
839 	}
840 	return 0;
841 }
842 #endif
843 
844 /*
845  * Customised RSA item verification routine. This is called when a signature
846  * is encountered requiring special handling. We currently only handle PSS.
847  */
848 static int
849 rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
850     X509_ALGOR *sigalg, ASN1_BIT_STRING *sig, EVP_PKEY *pkey)
851 {
852 	/* Sanity check: make sure it is PSS */
853 	if (OBJ_obj2nid(sigalg->algorithm) != EVP_PKEY_RSA_PSS) {
854 		RSAerror(RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
855 		return -1;
856 	}
857 	if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) {
858 		/* Carry on */
859 		return 2;
860 	}
861 	return -1;
862 }
863 
864 #ifndef OPENSSL_NO_CMS
865 static int
866 rsa_cms_sign(CMS_SignerInfo *si)
867 {
868 	int pad_mode = RSA_PKCS1_PADDING;
869 	X509_ALGOR *alg;
870 	EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
871 	ASN1_STRING *os = NULL;
872 
873 	CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
874 	if (pkctx) {
875 		if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
876 			return 0;
877 	}
878 	if (pad_mode == RSA_PKCS1_PADDING) {
879 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
880 		return 1;
881 	}
882 	/* We don't support it */
883 	if (pad_mode != RSA_PKCS1_PSS_PADDING)
884 		return 0;
885 	os = rsa_ctx_to_pss_string(pkctx);
886 	if (!os)
887 		return 0;
888 	X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os);
889 	return 1;
890 }
891 #endif
892 
893 static int
894 rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
895     X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *sig)
896 {
897 	EVP_PKEY_CTX *pkctx = ctx->pctx;
898 	int pad_mode;
899 
900 	if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
901 		return 0;
902 	if (pad_mode == RSA_PKCS1_PADDING)
903 		return 2;
904 	if (pad_mode == RSA_PKCS1_PSS_PADDING) {
905 		ASN1_STRING *os1 = NULL;
906 		os1 = rsa_ctx_to_pss_string(pkctx);
907 		if (!os1)
908 			return 0;
909 		/* Duplicate parameters if we have to */
910 		if (alg2) {
911 			ASN1_STRING *os2 = ASN1_STRING_dup(os1);
912 			if (!os2) {
913 				ASN1_STRING_free(os1);
914 				return 0;
915 			}
916 			X509_ALGOR_set0(alg2, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
917 			    V_ASN1_SEQUENCE, os2);
918 		}
919 		X509_ALGOR_set0(alg1, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
920 		    V_ASN1_SEQUENCE, os1);
921 		return 3;
922 	}
923 	return 2;
924 }
925 
926 static int
927 rsa_pkey_check(const EVP_PKEY *pkey)
928 {
929 	return RSA_check_key(pkey->pkey.rsa);
930 }
931 
932 #ifndef OPENSSL_NO_CMS
933 static RSA_OAEP_PARAMS *
934 rsa_oaep_decode(const X509_ALGOR *alg)
935 {
936 	RSA_OAEP_PARAMS *oaep;
937 
938 	oaep = ASN1_TYPE_unpack_sequence(&RSA_OAEP_PARAMS_it, alg->parameter);
939 	if (oaep == NULL)
940 		return NULL;
941 
942 	if (oaep->maskGenFunc != NULL) {
943 		oaep->maskHash = rsa_mgf1_decode(oaep->maskGenFunc);
944 		if (oaep->maskHash == NULL) {
945 			RSA_OAEP_PARAMS_free(oaep);
946 			return NULL;
947 		}
948 	}
949 	return oaep;
950 }
951 
952 static int
953 rsa_cms_decrypt(CMS_RecipientInfo *ri)
954 {
955 	EVP_PKEY_CTX *pkctx;
956 	X509_ALGOR *cmsalg;
957 	int nid;
958 	int rv = -1;
959 	unsigned char *label = NULL;
960 	int labellen = 0;
961 	const EVP_MD *mgf1md = NULL, *md = NULL;
962 	RSA_OAEP_PARAMS *oaep;
963 
964 	pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
965 	if (pkctx == NULL)
966 		return 0;
967 	if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
968 		return -1;
969 	nid = OBJ_obj2nid(cmsalg->algorithm);
970 	if (nid == NID_rsaEncryption)
971 		return 1;
972 	if (nid != NID_rsaesOaep) {
973 		RSAerror(RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
974 		return -1;
975 	}
976 	/* Decode OAEP parameters */
977 	oaep = rsa_oaep_decode(cmsalg);
978 
979 	if (oaep == NULL) {
980 		RSAerror(RSA_R_INVALID_OAEP_PARAMETERS);
981 		goto err;
982 	}
983 
984 	mgf1md = rsa_algor_to_md(oaep->maskHash);
985 	if (mgf1md == NULL)
986 		goto err;
987 	md = rsa_algor_to_md(oaep->hashFunc);
988 	if (md == NULL)
989 		goto err;
990 
991 	if (oaep->pSourceFunc != NULL) {
992 		X509_ALGOR *plab = oaep->pSourceFunc;
993 
994 		if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
995 			RSAerror(RSA_R_UNSUPPORTED_LABEL_SOURCE);
996 			goto err;
997 		}
998 		if (plab->parameter->type != V_ASN1_OCTET_STRING) {
999 			RSAerror(RSA_R_INVALID_LABEL);
1000 			goto err;
1001 		}
1002 
1003 		label = plab->parameter->value.octet_string->data;
1004 
1005 		/* Stop label being freed when OAEP parameters are freed */
1006 		/* XXX - this leaks label on error... */
1007 		plab->parameter->value.octet_string->data = NULL;
1008 		labellen = plab->parameter->value.octet_string->length;
1009 	}
1010 
1011 	if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
1012 		goto err;
1013 	if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
1014 		goto err;
1015 	if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
1016 		goto err;
1017 	if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
1018 		goto err;
1019 
1020 	rv = 1;
1021 
1022  err:
1023 	RSA_OAEP_PARAMS_free(oaep);
1024 	return rv;
1025 }
1026 
1027 static int
1028 rsa_cms_encrypt(CMS_RecipientInfo *ri)
1029 {
1030 	const EVP_MD *md, *mgf1md;
1031 	RSA_OAEP_PARAMS *oaep = NULL;
1032 	ASN1_STRING *os = NULL;
1033 	X509_ALGOR *alg;
1034 	EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
1035 	int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
1036 	unsigned char *label;
1037 
1038 	if (CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg) <= 0)
1039 		return 0;
1040 	if (pkctx) {
1041 		if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
1042 			return 0;
1043 	}
1044 	if (pad_mode == RSA_PKCS1_PADDING) {
1045 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
1046 		return 1;
1047 	}
1048 	/* Not supported */
1049 	if (pad_mode != RSA_PKCS1_OAEP_PADDING)
1050 		return 0;
1051 	if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
1052 		goto err;
1053 	if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
1054 		goto err;
1055 	labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
1056 	if (labellen < 0)
1057 		goto err;
1058 	oaep = RSA_OAEP_PARAMS_new();
1059 	if (oaep == NULL)
1060 		goto err;
1061 	if (!rsa_md_to_algor(&oaep->hashFunc, md))
1062 		goto err;
1063 	if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
1064 		goto err;
1065 	if (labellen > 0) {
1066 		ASN1_OCTET_STRING *los;
1067 		oaep->pSourceFunc = X509_ALGOR_new();
1068 		if (oaep->pSourceFunc == NULL)
1069 			goto err;
1070 		los = ASN1_OCTET_STRING_new();
1071 		if (los == NULL)
1072 			goto err;
1073 		if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
1074 			ASN1_OCTET_STRING_free(los);
1075 			goto err;
1076 		}
1077 		X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
1078 		    V_ASN1_OCTET_STRING, los);
1079 	}
1080 	/* create string with pss parameter encoding. */
1081 	if (!ASN1_item_pack(oaep, &RSA_OAEP_PARAMS_it, &os))
1082 		 goto err;
1083 	X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
1084 	os = NULL;
1085 	rv = 1;
1086  err:
1087 	RSA_OAEP_PARAMS_free(oaep);
1088 	ASN1_STRING_free(os);
1089 	return rv;
1090 }
1091 #endif
1092 
1093 const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = {
1094 	{
1095 		.pkey_id = EVP_PKEY_RSA,
1096 		.pkey_base_id = EVP_PKEY_RSA,
1097 		.pkey_flags = ASN1_PKEY_SIGPARAM_NULL,
1098 
1099 		.pem_str = "RSA",
1100 		.info = "OpenSSL RSA method",
1101 
1102 		.pub_decode = rsa_pub_decode,
1103 		.pub_encode = rsa_pub_encode,
1104 		.pub_cmp = rsa_pub_cmp,
1105 		.pub_print = rsa_pub_print,
1106 
1107 		.priv_decode = rsa_priv_decode,
1108 		.priv_encode = rsa_priv_encode,
1109 		.priv_print = rsa_priv_print,
1110 
1111 		.pkey_size = int_rsa_size,
1112 		.pkey_bits = rsa_bits,
1113 		.pkey_security_bits = rsa_security_bits,
1114 
1115 		.sig_print = rsa_sig_print,
1116 
1117 		.pkey_free = int_rsa_free,
1118 		.pkey_ctrl = rsa_pkey_ctrl,
1119 		.old_priv_decode = old_rsa_priv_decode,
1120 		.old_priv_encode = old_rsa_priv_encode,
1121 		.item_verify = rsa_item_verify,
1122 		.item_sign = rsa_item_sign,
1123 
1124 		.pkey_check = rsa_pkey_check,
1125 	},
1126 
1127 	{
1128 		.pkey_id = EVP_PKEY_RSA2,
1129 		.pkey_base_id = EVP_PKEY_RSA,
1130 		.pkey_flags = ASN1_PKEY_ALIAS,
1131 
1132 		.pkey_check = rsa_pkey_check,
1133 	},
1134 };
1135 
1136 const EVP_PKEY_ASN1_METHOD rsa_pss_asn1_meth = {
1137 	.pkey_id = EVP_PKEY_RSA_PSS,
1138 	.pkey_base_id = EVP_PKEY_RSA_PSS,
1139 	.pkey_flags = ASN1_PKEY_SIGPARAM_NULL,
1140 
1141 	.pem_str = "RSA-PSS",
1142 	.info = "OpenSSL RSA-PSS method",
1143 
1144 	.pub_decode = rsa_pub_decode,
1145 	.pub_encode = rsa_pub_encode,
1146 	.pub_cmp = rsa_pub_cmp,
1147 	.pub_print = rsa_pub_print,
1148 
1149 	.priv_decode = rsa_priv_decode,
1150 	.priv_encode = rsa_priv_encode,
1151 	.priv_print = rsa_priv_print,
1152 
1153 	.pkey_size = int_rsa_size,
1154 	.pkey_bits = rsa_bits,
1155 	.pkey_security_bits = rsa_security_bits,
1156 
1157 	.sig_print = rsa_sig_print,
1158 
1159 	.pkey_free = int_rsa_free,
1160 	.pkey_ctrl = rsa_pkey_ctrl,
1161 	.item_verify = rsa_item_verify,
1162 	.item_sign = rsa_item_sign
1163 };
1164