xref: /openbsd-src/lib/libcrypto/rsa/rsa_ameth.c (revision f90ef06a3045119dcc88b72d8b98ca60e3c00d5a)
1 /* $OpenBSD: rsa_ameth.c,v 1.33 2023/08/12 08:02:43 tb Exp $ */
2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58 
59 #include <stdio.h>
60 
61 #include <openssl/opensslconf.h>
62 
63 #include <openssl/asn1t.h>
64 #include <openssl/bn.h>
65 #include <openssl/cms.h>
66 #include <openssl/err.h>
67 #include <openssl/rsa.h>
68 #include <openssl/x509.h>
69 
70 #include "asn1_local.h"
71 #include "bn_local.h"
72 #include "cryptlib.h"
73 #include "evp_local.h"
74 #include "rsa_local.h"
75 
76 #ifndef OPENSSL_NO_CMS
77 static int rsa_cms_sign(CMS_SignerInfo *si);
78 static int rsa_cms_verify(CMS_SignerInfo *si);
79 static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
80 static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
81 #endif
82 
83 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg);
84 
85 /* Set any parameters associated with pkey */
86 static int
87 rsa_param_encode(const EVP_PKEY *pkey, ASN1_STRING **pstr, int *pstrtype)
88 {
89 	const RSA *rsa = pkey->pkey.rsa;
90 
91 	*pstr = NULL;
92 
93 	/* If RSA it's just NULL type */
94 	if (pkey->ameth->pkey_id != EVP_PKEY_RSA_PSS) {
95 		*pstrtype = V_ASN1_NULL;
96 		return 1;
97 	}
98 
99 	/* If no PSS parameters we omit parameters entirely */
100 	if (rsa->pss == NULL) {
101 		*pstrtype = V_ASN1_UNDEF;
102 		return 1;
103 	}
104 
105 	/* Encode PSS parameters */
106 	if (ASN1_item_pack(rsa->pss, &RSA_PSS_PARAMS_it, pstr) == NULL)
107 		return 0;
108 
109 	*pstrtype = V_ASN1_SEQUENCE;
110 	return 1;
111 }
112 
113 /* Decode any parameters and set them in RSA structure */
114 static int
115 rsa_param_decode(RSA *rsa, const X509_ALGOR *alg)
116 {
117 	const ASN1_OBJECT *algoid;
118 	const void *algp;
119 	int algptype;
120 
121 	X509_ALGOR_get0(&algoid, &algptype, &algp, alg);
122 	if (OBJ_obj2nid(algoid) != EVP_PKEY_RSA_PSS)
123 		return 1;
124 	if (algptype == V_ASN1_UNDEF)
125 		return 1;
126 	if (algptype != V_ASN1_SEQUENCE) {
127 		RSAerror(RSA_R_INVALID_PSS_PARAMETERS);
128 		return 0;
129 	}
130 	rsa->pss = rsa_pss_decode(alg);
131 	if (rsa->pss == NULL)
132 		return 0;
133 	return 1;
134 }
135 
136 static int
137 rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
138 {
139 	ASN1_STRING *str = NULL;
140 	int strtype;
141 	unsigned char *penc = NULL;
142 	int penclen = 0;
143 	ASN1_OBJECT *aobj;
144 
145 	if (!rsa_param_encode(pkey, &str, &strtype))
146 		goto err;
147 	if ((penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc)) <= 0) {
148 		penclen = 0;
149 		goto err;
150 	}
151 	if ((aobj = OBJ_nid2obj(pkey->ameth->pkey_id)) == NULL)
152 		goto err;
153 	if (!X509_PUBKEY_set0_param(pk, aobj, strtype, str, penc, penclen))
154 		goto err;
155 
156 	return 1;
157 
158  err:
159 	ASN1_STRING_free(str);
160 	freezero(penc, penclen);
161 
162 	return 0;
163 }
164 
165 static int
166 rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
167 {
168 	const unsigned char *p;
169 	int pklen;
170 	X509_ALGOR *alg;
171 	RSA *rsa = NULL;
172 
173 	if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, &alg, pubkey))
174 		return 0;
175 	if ((rsa = d2i_RSAPublicKey(NULL, &p, pklen)) == NULL) {
176 		RSAerror(ERR_R_RSA_LIB);
177 		return 0;
178 	}
179 	if (!rsa_param_decode(rsa, alg)) {
180 		RSA_free(rsa);
181 		return 0;
182 	}
183 	if (!EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa)) {
184 		RSA_free(rsa);
185 		return 0;
186 	}
187 	return 1;
188 }
189 
190 static int
191 rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
192 {
193 	if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0 ||
194 	    BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
195 		return 0;
196 
197 	return 1;
198 }
199 
200 static int
201 old_rsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
202 {
203 	RSA *rsa;
204 
205 	if ((rsa = d2i_RSAPrivateKey(NULL, pder, derlen)) == NULL) {
206 		RSAerror(ERR_R_RSA_LIB);
207 		return 0;
208 	}
209 	EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
210 	return 1;
211 }
212 
213 static int
214 old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
215 {
216 	return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
217 }
218 
219 static int
220 rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
221 {
222 	ASN1_STRING *str = NULL;
223 	ASN1_OBJECT *aobj;
224 	int strtype;
225 	unsigned char *rk = NULL;
226 	int rklen = 0;
227 
228 	if (!rsa_param_encode(pkey, &str, &strtype))
229 		goto err;
230 	if ((rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk)) <= 0) {
231 		RSAerror(ERR_R_MALLOC_FAILURE);
232 		rklen = 0;
233 		goto err;
234 	}
235 	if ((aobj = OBJ_nid2obj(pkey->ameth->pkey_id)) == NULL)
236 		goto err;
237 	if (!PKCS8_pkey_set0(p8, aobj, 0, strtype, str, rk, rklen)) {
238 		RSAerror(ERR_R_MALLOC_FAILURE);
239 		goto err;
240 	}
241 
242 	return 1;
243 
244  err:
245 	ASN1_STRING_free(str);
246 	freezero(rk, rklen);
247 
248 	return 0;
249 }
250 
251 static int
252 rsa_priv_decode(EVP_PKEY *pkey, const PKCS8_PRIV_KEY_INFO *p8)
253 {
254 	const unsigned char *p;
255 	RSA *rsa;
256 	int pklen;
257 	const X509_ALGOR *alg;
258 
259 	if (!PKCS8_pkey_get0(NULL, &p, &pklen, &alg, p8))
260 		return 0;
261 	rsa = d2i_RSAPrivateKey(NULL, &p, pklen);
262 	if (rsa == NULL) {
263 		RSAerror(ERR_R_RSA_LIB);
264 		return 0;
265 	}
266 	if (!rsa_param_decode(rsa, alg)) {
267 		RSA_free(rsa);
268 		return 0;
269 	}
270 	EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
271 
272 	return 1;
273 }
274 
275 static int
276 rsa_size(const EVP_PKEY *pkey)
277 {
278 	return RSA_size(pkey->pkey.rsa);
279 }
280 
281 static int
282 rsa_bits(const EVP_PKEY *pkey)
283 {
284 	return BN_num_bits(pkey->pkey.rsa->n);
285 }
286 
287 static int
288 rsa_security_bits(const EVP_PKEY *pkey)
289 {
290 	return RSA_security_bits(pkey->pkey.rsa);
291 }
292 
293 static void
294 rsa_free(EVP_PKEY *pkey)
295 {
296 	RSA_free(pkey->pkey.rsa);
297 }
298 
299 static X509_ALGOR *
300 rsa_mgf1_decode(X509_ALGOR *alg)
301 {
302 	if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
303 		return NULL;
304 
305 	return ASN1_TYPE_unpack_sequence(&X509_ALGOR_it, alg->parameter);
306 }
307 
308 static RSA_PSS_PARAMS *
309 rsa_pss_decode(const X509_ALGOR *alg)
310 {
311 	RSA_PSS_PARAMS *pss;
312 
313 	pss = ASN1_TYPE_unpack_sequence(&RSA_PSS_PARAMS_it, alg->parameter);
314 	if (pss == NULL)
315 		return NULL;
316 
317 	if (pss->maskGenAlgorithm != NULL) {
318 		pss->maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
319 		if (pss->maskHash == NULL) {
320 			RSA_PSS_PARAMS_free(pss);
321 			return NULL;
322 		}
323 	}
324 
325 	return pss;
326 }
327 
328 static int
329 rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss, int indent)
330 {
331 	int rv = 0;
332 	X509_ALGOR *maskHash = NULL;
333 
334 	if (!BIO_indent(bp, indent, 128))
335 		goto err;
336 	if (pss_key) {
337 		if (pss == NULL) {
338 			if (BIO_puts(bp, "No PSS parameter restrictions\n") <= 0)
339 				return 0;
340 			return 1;
341 		} else {
342 			if (BIO_puts(bp, "PSS parameter restrictions:") <= 0)
343 				return 0;
344 		}
345 	} else if (pss == NULL) {
346 		if (BIO_puts(bp,"(INVALID PSS PARAMETERS)\n") <= 0)
347 			return 0;
348 		return 1;
349 	}
350 	if (BIO_puts(bp, "\n") <= 0)
351 		goto err;
352 	if (pss_key)
353 		indent += 2;
354 	if (!BIO_indent(bp, indent, 128))
355 		goto err;
356 	if (BIO_puts(bp, "Hash Algorithm: ") <= 0)
357 		goto err;
358 
359 	if (pss->hashAlgorithm) {
360 		if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0)
361 			goto err;
362 	} else if (BIO_puts(bp, "sha1 (default)") <= 0) {
363 		goto err;
364 	}
365 
366 	if (BIO_puts(bp, "\n") <= 0)
367 		goto err;
368 
369 	if (!BIO_indent(bp, indent, 128))
370 		goto err;
371 
372 	if (BIO_puts(bp, "Mask Algorithm: ") <= 0)
373 		goto err;
374 	if (pss->maskGenAlgorithm) {
375 		if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0)
376 			goto err;
377 		if (BIO_puts(bp, " with ") <= 0)
378 			goto err;
379 		maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
380 		if (maskHash != NULL) {
381 			if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
382 				goto err;
383 		} else if (BIO_puts(bp, "INVALID") <= 0) {
384 			goto err;
385 		}
386 	} else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0) {
387 		goto err;
388 	}
389 	BIO_puts(bp, "\n");
390 
391 	if (!BIO_indent(bp, indent, 128))
392 		goto err;
393 	if (BIO_printf(bp, "%s Salt Length: 0x", pss_key ? "Minimum" : "") <= 0)
394 		goto err;
395 	if (pss->saltLength) {
396 		if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0)
397 			goto err;
398 	} else if (BIO_puts(bp, "14 (default)") <= 0) {
399 		goto err;
400 	}
401 	BIO_puts(bp, "\n");
402 
403 	if (!BIO_indent(bp, indent, 128))
404 		goto err;
405 	if (BIO_puts(bp, "Trailer Field: 0x") <= 0)
406 		goto err;
407 	if (pss->trailerField) {
408 		if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0)
409 			goto err;
410 	} else if (BIO_puts(bp, "BC (default)") <= 0) {
411 		goto err;
412 	}
413 	BIO_puts(bp, "\n");
414 
415 	rv = 1;
416 
417  err:
418 	X509_ALGOR_free(maskHash);
419 	return rv;
420 
421 }
422 
423 static int
424 pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
425 {
426 	const RSA *x = pkey->pkey.rsa;
427 	char *str;
428 	const char *s;
429 	int ret = 0, mod_len = 0;
430 
431 	if (x->n != NULL)
432 		mod_len = BN_num_bits(x->n);
433 
434 	if (!BIO_indent(bp, off, 128))
435 		goto err;
436 
437 	if (BIO_printf(bp, "%s ", pkey_is_pss(pkey) ?  "RSA-PSS" : "RSA") <= 0)
438 		goto err;
439 
440 	if (priv && x->d != NULL) {
441 		if (BIO_printf(bp, "Private-Key: (%d bit)\n", mod_len) <= 0)
442 			goto err;
443 		str = "modulus:";
444 		s = "publicExponent:";
445 	} else {
446 		if (BIO_printf(bp, "Public-Key: (%d bit)\n", mod_len) <= 0)
447 			goto err;
448 		str = "Modulus:";
449 		s = "Exponent:";
450 	}
451 	if (!bn_printf(bp, x->n, off, "%s", str))
452 		goto err;
453 	if (!bn_printf(bp, x->e, off, "%s", s))
454 		goto err;
455 	if (priv) {
456 		if (!bn_printf(bp, x->d, off, "privateExponent:"))
457 			goto err;
458 		if (!bn_printf(bp, x->p, off, "prime1:"))
459 			goto err;
460 		if (!bn_printf(bp, x->q, off, "prime2:"))
461 			goto err;
462 		if (!bn_printf(bp, x->dmp1, off, "exponent1:"))
463 			goto err;
464 		if (!bn_printf(bp, x->dmq1, off, "exponent2:"))
465 			goto err;
466 		if (!bn_printf(bp, x->iqmp, off, "coefficient:"))
467 			goto err;
468 	}
469 	if (pkey_is_pss(pkey) && !rsa_pss_param_print(bp, 1, x->pss, off))
470 		goto err;
471 	ret = 1;
472  err:
473 	return ret;
474 }
475 
476 static int
477 rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent, ASN1_PCTX *ctx)
478 {
479 	return pkey_rsa_print(bp, pkey, indent, 0);
480 }
481 
482 static int
483 rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent, ASN1_PCTX *ctx)
484 {
485 	return pkey_rsa_print(bp, pkey, indent, 1);
486 }
487 
488 static int
489 rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg, const ASN1_STRING *sig,
490     int indent, ASN1_PCTX *pctx)
491 {
492 	if (OBJ_obj2nid(sigalg->algorithm) == EVP_PKEY_RSA_PSS) {
493 		int rv;
494 		RSA_PSS_PARAMS *pss = rsa_pss_decode(sigalg);
495 
496 		rv = rsa_pss_param_print(bp, 0, pss, indent);
497 		RSA_PSS_PARAMS_free(pss);
498 		if (!rv)
499 			return 0;
500 	} else if (!sig && BIO_puts(bp, "\n") <= 0) {
501 		return 0;
502 	}
503 	if (sig)
504 		return X509_signature_dump(bp, sig, indent);
505 	return 1;
506 }
507 
508 static int
509 rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
510 {
511 	X509_ALGOR *alg = NULL;
512 	const EVP_MD *md;
513 	const EVP_MD *mgf1md;
514 	int min_saltlen;
515 
516 	switch (op) {
517 	case ASN1_PKEY_CTRL_PKCS7_SIGN:
518 		if (arg1 == 0)
519 			PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
520 		break;
521 
522 	case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
523 		if (pkey_is_pss(pkey))
524 			return -2;
525 		if (arg1 == 0)
526 			PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
527 		break;
528 #ifndef OPENSSL_NO_CMS
529 	case ASN1_PKEY_CTRL_CMS_SIGN:
530 		if (arg1 == 0)
531 			return rsa_cms_sign(arg2);
532 		else if (arg1 == 1)
533 			return rsa_cms_verify(arg2);
534 		break;
535 
536 	case ASN1_PKEY_CTRL_CMS_ENVELOPE:
537 		if (pkey_is_pss(pkey))
538 			return -2;
539 		if (arg1 == 0)
540 			return rsa_cms_encrypt(arg2);
541 		else if (arg1 == 1)
542 			return rsa_cms_decrypt(arg2);
543 		break;
544 
545 	case ASN1_PKEY_CTRL_CMS_RI_TYPE:
546 		if (pkey_is_pss(pkey))
547 			return -2;
548 		*(int *)arg2 = CMS_RECIPINFO_TRANS;
549 		return 1;
550 #endif
551 
552 	case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
553 		if (pkey->pkey.rsa->pss != NULL) {
554 			if (!rsa_pss_get_param(pkey->pkey.rsa->pss, &md, &mgf1md,
555 			    &min_saltlen)) {
556 				RSAerror(ERR_R_INTERNAL_ERROR);
557 				return 0;
558 			}
559 			*(int *)arg2 = EVP_MD_type(md);
560 			/* Return of 2 indicates this MD is mandatory */
561 			return 2;
562 		}
563 		*(int *)arg2 = NID_sha256;
564 		return 1;
565 
566 	default:
567 		return -2;
568 	}
569 
570 	if (alg)
571 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
572 		    V_ASN1_NULL, 0);
573 
574 	return 1;
575 }
576 
577 /* Allocate and set algorithm ID from EVP_MD, defaults to SHA1. */
578 static int
579 rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
580 {
581 	if (md == NULL || EVP_MD_type(md) == NID_sha1)
582 		return 1;
583 	*palg = X509_ALGOR_new();
584 	if (*palg == NULL)
585 		return 0;
586 	X509_ALGOR_set_md(*palg, md);
587 	return 1;
588 }
589 
590 /* Allocate and set MGF1 algorithm ID from EVP_MD. */
591 static int
592 rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
593 {
594 	X509_ALGOR *algtmp = NULL;
595 	ASN1_STRING *stmp = NULL;
596 
597 	*palg = NULL;
598 	if (mgf1md == NULL || EVP_MD_type(mgf1md) == NID_sha1)
599 		return 1;
600 	/* need to embed algorithm ID inside another */
601 	if (!rsa_md_to_algor(&algtmp, mgf1md))
602 		goto err;
603 	if (ASN1_item_pack(algtmp, &X509_ALGOR_it, &stmp) == NULL)
604 		 goto err;
605 	*palg = X509_ALGOR_new();
606 	if (*palg == NULL)
607 		goto err;
608 	X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
609 	stmp = NULL;
610  err:
611 	ASN1_STRING_free(stmp);
612 	X509_ALGOR_free(algtmp);
613 	if (*palg)
614 		return 1;
615 	return 0;
616 }
617 
618 /* Convert algorithm ID to EVP_MD, defaults to SHA1. */
619 static const EVP_MD *
620 rsa_algor_to_md(X509_ALGOR *alg)
621 {
622 	const EVP_MD *md;
623 
624 	if (!alg)
625 		return EVP_sha1();
626 	md = EVP_get_digestbyobj(alg->algorithm);
627 	if (md == NULL)
628 		RSAerror(RSA_R_UNKNOWN_DIGEST);
629 	return md;
630 }
631 
632 /*
633  * Convert EVP_PKEY_CTX in PSS mode into corresponding algorithm parameter,
634  * suitable for setting an AlgorithmIdentifier.
635  */
636 static RSA_PSS_PARAMS *
637 rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
638 {
639 	const EVP_MD *sigmd, *mgf1md;
640 	EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
641 	int saltlen;
642 
643 	if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
644 		return NULL;
645 	if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
646 		return NULL;
647 	if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
648 		return NULL;
649 	if (saltlen == -1) {
650 		saltlen = EVP_MD_size(sigmd);
651 	} else if (saltlen == -2 || saltlen == -3) {
652 		saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
653 		if ((EVP_PKEY_bits(pk) & 0x7) == 1)
654 			saltlen--;
655 		if (saltlen < 0)
656 			return NULL;
657 	}
658 
659 	return rsa_pss_params_create(sigmd, mgf1md, saltlen);
660 }
661 
662 RSA_PSS_PARAMS *
663 rsa_pss_params_create(const EVP_MD *sigmd, const EVP_MD *mgf1md, int saltlen)
664 {
665 	RSA_PSS_PARAMS *pss = RSA_PSS_PARAMS_new();
666 
667 	if (pss == NULL)
668 		goto err;
669 	if (saltlen != 20) {
670 		pss->saltLength = ASN1_INTEGER_new();
671 		if (pss->saltLength == NULL)
672 			goto err;
673 		if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
674 			goto err;
675 	}
676 	if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
677 		goto err;
678 	if (mgf1md == NULL)
679 		mgf1md = sigmd;
680 	if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
681 		goto err;
682 	if (!rsa_md_to_algor(&pss->maskHash, mgf1md))
683 		goto err;
684 	return pss;
685  err:
686 	RSA_PSS_PARAMS_free(pss);
687 	return NULL;
688 }
689 
690 static ASN1_STRING *
691 rsa_ctx_to_pss_string(EVP_PKEY_CTX *pkctx)
692 {
693 	RSA_PSS_PARAMS *pss = rsa_ctx_to_pss(pkctx);
694 	ASN1_STRING *os;
695 
696 	if (pss == NULL)
697 		return NULL;
698 
699 	os = ASN1_item_pack(pss, &RSA_PSS_PARAMS_it, NULL);
700 	RSA_PSS_PARAMS_free(pss);
701 	return os;
702 }
703 
704 /*
705  * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL
706  * then the EVP_MD_CTX is setup and initialised. If it is NULL parameters are
707  * passed to pkctx instead.
708  */
709 
710 static int
711 rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
712     X509_ALGOR *sigalg, EVP_PKEY *pkey)
713 {
714 	int rv = -1;
715 	int saltlen;
716 	const EVP_MD *mgf1md = NULL, *md = NULL;
717 	RSA_PSS_PARAMS *pss;
718 
719 	/* Sanity check: make sure it is PSS */
720 	if (OBJ_obj2nid(sigalg->algorithm) != EVP_PKEY_RSA_PSS) {
721 		RSAerror(RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
722 		return -1;
723 	}
724 	/* Decode PSS parameters */
725 	pss = rsa_pss_decode(sigalg);
726 
727 	if (!rsa_pss_get_param(pss, &md, &mgf1md, &saltlen)) {
728 		RSAerror(RSA_R_INVALID_PSS_PARAMETERS);
729 		goto err;
730 	}
731 
732 	/* We have all parameters now set up context */
733 	if (pkey) {
734 		if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
735 			goto err;
736 	} else {
737 		const EVP_MD *checkmd;
738 		if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0)
739 			goto err;
740 		if (EVP_MD_type(md) != EVP_MD_type(checkmd)) {
741 			RSAerror(RSA_R_DIGEST_DOES_NOT_MATCH);
742 			goto err;
743 		}
744 	}
745 
746 	if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0)
747 		goto err;
748 
749 	if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0)
750 		goto err;
751 
752 	if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
753 		goto err;
754 	/* Carry on */
755 	rv = 1;
756 
757  err:
758 	RSA_PSS_PARAMS_free(pss);
759 	return rv;
760 }
761 
762 int
763 rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,
764     const EVP_MD **pmgf1md, int *psaltlen)
765 {
766 	if (pss == NULL)
767 		return 0;
768 	*pmd = rsa_algor_to_md(pss->hashAlgorithm);
769 	if (*pmd == NULL)
770 		return 0;
771 	*pmgf1md = rsa_algor_to_md(pss->maskHash);
772 	if (*pmgf1md == NULL)
773 		return 0;
774 	if (pss->saltLength) {
775 		*psaltlen = ASN1_INTEGER_get(pss->saltLength);
776 		if (*psaltlen < 0) {
777 			RSAerror(RSA_R_INVALID_SALT_LENGTH);
778 			return 0;
779 		}
780 	} else {
781 		*psaltlen = 20;
782 	}
783 
784 	/*
785 	 * low-level routines support only trailer field 0xbc (value 1) and
786 	 * PKCS#1 says we should reject any other value anyway.
787 	 */
788 	if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
789 		RSAerror(RSA_R_INVALID_TRAILER);
790 		return 0;
791 	}
792 
793 	return 1;
794 }
795 
796 #ifndef OPENSSL_NO_CMS
797 static int
798 rsa_cms_verify(CMS_SignerInfo *si)
799 {
800 	int nid, nid2;
801 	X509_ALGOR *alg;
802 	EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
803 
804 	CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
805 	nid = OBJ_obj2nid(alg->algorithm);
806 	if (nid == EVP_PKEY_RSA_PSS)
807 		return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
808 	/* Only PSS allowed for PSS keys */
809 	if (pkey_ctx_is_pss(pkctx)) {
810 		RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
811 		return 0;
812 	}
813 	if (nid == NID_rsaEncryption)
814 		return 1;
815 	/* Workaround for some implementation that use a signature OID */
816 	if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
817 		if (nid2 == NID_rsaEncryption)
818 			return 1;
819 	}
820 	return 0;
821 }
822 #endif
823 
824 /*
825  * Customised RSA item verification routine. This is called when a signature
826  * is encountered requiring special handling. We currently only handle PSS.
827  */
828 static int
829 rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
830     X509_ALGOR *sigalg, ASN1_BIT_STRING *sig, EVP_PKEY *pkey)
831 {
832 	/* Sanity check: make sure it is PSS */
833 	if (OBJ_obj2nid(sigalg->algorithm) != EVP_PKEY_RSA_PSS) {
834 		RSAerror(RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
835 		return -1;
836 	}
837 	if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) {
838 		/* Carry on */
839 		return 2;
840 	}
841 	return -1;
842 }
843 
844 #ifndef OPENSSL_NO_CMS
845 static int
846 rsa_cms_sign(CMS_SignerInfo *si)
847 {
848 	int pad_mode = RSA_PKCS1_PADDING;
849 	X509_ALGOR *alg;
850 	EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
851 	ASN1_STRING *os = NULL;
852 
853 	CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
854 	if (pkctx) {
855 		if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
856 			return 0;
857 	}
858 	if (pad_mode == RSA_PKCS1_PADDING) {
859 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
860 		return 1;
861 	}
862 	/* We don't support it */
863 	if (pad_mode != RSA_PKCS1_PSS_PADDING)
864 		return 0;
865 	os = rsa_ctx_to_pss_string(pkctx);
866 	if (!os)
867 		return 0;
868 	X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os);
869 	return 1;
870 }
871 #endif
872 
873 static int
874 rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
875     X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *sig)
876 {
877 	EVP_PKEY_CTX *pkctx = ctx->pctx;
878 	int pad_mode;
879 
880 	if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
881 		return 0;
882 	if (pad_mode == RSA_PKCS1_PADDING)
883 		return 2;
884 	if (pad_mode == RSA_PKCS1_PSS_PADDING) {
885 		ASN1_STRING *os1 = NULL;
886 		os1 = rsa_ctx_to_pss_string(pkctx);
887 		if (!os1)
888 			return 0;
889 		/* Duplicate parameters if we have to */
890 		if (alg2) {
891 			ASN1_STRING *os2 = ASN1_STRING_dup(os1);
892 			if (!os2) {
893 				ASN1_STRING_free(os1);
894 				return 0;
895 			}
896 			X509_ALGOR_set0(alg2, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
897 			    V_ASN1_SEQUENCE, os2);
898 		}
899 		X509_ALGOR_set0(alg1, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
900 		    V_ASN1_SEQUENCE, os1);
901 		return 3;
902 	}
903 	return 2;
904 }
905 
906 static int
907 rsa_pkey_check(const EVP_PKEY *pkey)
908 {
909 	return RSA_check_key(pkey->pkey.rsa);
910 }
911 
912 #ifndef OPENSSL_NO_CMS
913 static RSA_OAEP_PARAMS *
914 rsa_oaep_decode(const X509_ALGOR *alg)
915 {
916 	RSA_OAEP_PARAMS *oaep;
917 
918 	oaep = ASN1_TYPE_unpack_sequence(&RSA_OAEP_PARAMS_it, alg->parameter);
919 	if (oaep == NULL)
920 		return NULL;
921 
922 	if (oaep->maskGenFunc != NULL) {
923 		oaep->maskHash = rsa_mgf1_decode(oaep->maskGenFunc);
924 		if (oaep->maskHash == NULL) {
925 			RSA_OAEP_PARAMS_free(oaep);
926 			return NULL;
927 		}
928 	}
929 	return oaep;
930 }
931 
932 static int
933 rsa_cms_decrypt(CMS_RecipientInfo *ri)
934 {
935 	EVP_PKEY_CTX *pkctx;
936 	X509_ALGOR *cmsalg;
937 	int nid;
938 	int rv = -1;
939 	unsigned char *label = NULL;
940 	int labellen = 0;
941 	const EVP_MD *mgf1md = NULL, *md = NULL;
942 	RSA_OAEP_PARAMS *oaep;
943 
944 	pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
945 	if (pkctx == NULL)
946 		return 0;
947 	if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
948 		return -1;
949 	nid = OBJ_obj2nid(cmsalg->algorithm);
950 	if (nid == NID_rsaEncryption)
951 		return 1;
952 	if (nid != NID_rsaesOaep) {
953 		RSAerror(RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
954 		return -1;
955 	}
956 	/* Decode OAEP parameters */
957 	oaep = rsa_oaep_decode(cmsalg);
958 
959 	if (oaep == NULL) {
960 		RSAerror(RSA_R_INVALID_OAEP_PARAMETERS);
961 		goto err;
962 	}
963 
964 	mgf1md = rsa_algor_to_md(oaep->maskHash);
965 	if (mgf1md == NULL)
966 		goto err;
967 	md = rsa_algor_to_md(oaep->hashFunc);
968 	if (md == NULL)
969 		goto err;
970 
971 	if (oaep->pSourceFunc != NULL) {
972 		X509_ALGOR *plab = oaep->pSourceFunc;
973 
974 		if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
975 			RSAerror(RSA_R_UNSUPPORTED_LABEL_SOURCE);
976 			goto err;
977 		}
978 		if (plab->parameter->type != V_ASN1_OCTET_STRING) {
979 			RSAerror(RSA_R_INVALID_LABEL);
980 			goto err;
981 		}
982 
983 		label = plab->parameter->value.octet_string->data;
984 
985 		/* Stop label being freed when OAEP parameters are freed */
986 		/* XXX - this leaks label on error... */
987 		plab->parameter->value.octet_string->data = NULL;
988 		labellen = plab->parameter->value.octet_string->length;
989 	}
990 
991 	if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
992 		goto err;
993 	if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
994 		goto err;
995 	if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
996 		goto err;
997 	if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
998 		goto err;
999 
1000 	rv = 1;
1001 
1002  err:
1003 	RSA_OAEP_PARAMS_free(oaep);
1004 	return rv;
1005 }
1006 
1007 static int
1008 rsa_cms_encrypt(CMS_RecipientInfo *ri)
1009 {
1010 	const EVP_MD *md, *mgf1md;
1011 	RSA_OAEP_PARAMS *oaep = NULL;
1012 	ASN1_STRING *os = NULL;
1013 	X509_ALGOR *alg;
1014 	EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
1015 	int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
1016 	unsigned char *label;
1017 
1018 	if (CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg) <= 0)
1019 		return 0;
1020 	if (pkctx) {
1021 		if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
1022 			return 0;
1023 	}
1024 	if (pad_mode == RSA_PKCS1_PADDING) {
1025 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
1026 		return 1;
1027 	}
1028 	/* Not supported */
1029 	if (pad_mode != RSA_PKCS1_OAEP_PADDING)
1030 		return 0;
1031 	if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
1032 		goto err;
1033 	if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
1034 		goto err;
1035 	labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
1036 	if (labellen < 0)
1037 		goto err;
1038 	oaep = RSA_OAEP_PARAMS_new();
1039 	if (oaep == NULL)
1040 		goto err;
1041 	if (!rsa_md_to_algor(&oaep->hashFunc, md))
1042 		goto err;
1043 	if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
1044 		goto err;
1045 	if (labellen > 0) {
1046 		ASN1_OCTET_STRING *los;
1047 		oaep->pSourceFunc = X509_ALGOR_new();
1048 		if (oaep->pSourceFunc == NULL)
1049 			goto err;
1050 		los = ASN1_OCTET_STRING_new();
1051 		if (los == NULL)
1052 			goto err;
1053 		if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
1054 			ASN1_OCTET_STRING_free(los);
1055 			goto err;
1056 		}
1057 		X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
1058 		    V_ASN1_OCTET_STRING, los);
1059 	}
1060 	/* create string with pss parameter encoding. */
1061 	if (!ASN1_item_pack(oaep, &RSA_OAEP_PARAMS_it, &os))
1062 		 goto err;
1063 	X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
1064 	os = NULL;
1065 	rv = 1;
1066  err:
1067 	RSA_OAEP_PARAMS_free(oaep);
1068 	ASN1_STRING_free(os);
1069 	return rv;
1070 }
1071 #endif
1072 
1073 const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = {
1074 	{
1075 		.pkey_id = EVP_PKEY_RSA,
1076 		.pkey_base_id = EVP_PKEY_RSA,
1077 		.pkey_flags = ASN1_PKEY_SIGPARAM_NULL,
1078 
1079 		.pem_str = "RSA",
1080 		.info = "OpenSSL RSA method",
1081 
1082 		.pub_decode = rsa_pub_decode,
1083 		.pub_encode = rsa_pub_encode,
1084 		.pub_cmp = rsa_pub_cmp,
1085 		.pub_print = rsa_pub_print,
1086 
1087 		.priv_decode = rsa_priv_decode,
1088 		.priv_encode = rsa_priv_encode,
1089 		.priv_print = rsa_priv_print,
1090 
1091 		.pkey_size = rsa_size,
1092 		.pkey_bits = rsa_bits,
1093 		.pkey_security_bits = rsa_security_bits,
1094 
1095 		.sig_print = rsa_sig_print,
1096 
1097 		.pkey_free = rsa_free,
1098 		.pkey_ctrl = rsa_pkey_ctrl,
1099 		.old_priv_decode = old_rsa_priv_decode,
1100 		.old_priv_encode = old_rsa_priv_encode,
1101 		.item_verify = rsa_item_verify,
1102 		.item_sign = rsa_item_sign,
1103 
1104 		.pkey_check = rsa_pkey_check,
1105 	},
1106 
1107 	{
1108 		.pkey_id = EVP_PKEY_RSA2,
1109 		.pkey_base_id = EVP_PKEY_RSA,
1110 		.pkey_flags = ASN1_PKEY_ALIAS,
1111 
1112 		.pkey_check = rsa_pkey_check,
1113 	},
1114 };
1115 
1116 const EVP_PKEY_ASN1_METHOD rsa_pss_asn1_meth = {
1117 	.pkey_id = EVP_PKEY_RSA_PSS,
1118 	.pkey_base_id = EVP_PKEY_RSA_PSS,
1119 	.pkey_flags = ASN1_PKEY_SIGPARAM_NULL,
1120 
1121 	.pem_str = "RSA-PSS",
1122 	.info = "OpenSSL RSA-PSS method",
1123 
1124 	.pub_decode = rsa_pub_decode,
1125 	.pub_encode = rsa_pub_encode,
1126 	.pub_cmp = rsa_pub_cmp,
1127 	.pub_print = rsa_pub_print,
1128 
1129 	.priv_decode = rsa_priv_decode,
1130 	.priv_encode = rsa_priv_encode,
1131 	.priv_print = rsa_priv_print,
1132 
1133 	.pkey_size = rsa_size,
1134 	.pkey_bits = rsa_bits,
1135 	.pkey_security_bits = rsa_security_bits,
1136 
1137 	.sig_print = rsa_sig_print,
1138 
1139 	.pkey_free = rsa_free,
1140 	.pkey_ctrl = rsa_pkey_ctrl,
1141 	.item_verify = rsa_item_verify,
1142 	.item_sign = rsa_item_sign
1143 };
1144