xref: /openbsd-src/lib/libcrypto/rsa/rsa_oaep.c (revision a8913c44aee6c78b4770e56ab6afb429afabee6d)
1 /* $OpenBSD: rsa_oaep.c,v 1.21 2014/07/10 13:58:23 jsing Exp $ */
2 /* Written by Ulf Moeller. This software is distributed on an "AS IS"
3    basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. */
4 
5 /* EME-OAEP as defined in RFC 2437 (PKCS #1 v2.0) */
6 
7 /* See Victor Shoup, "OAEP reconsidered," Nov. 2000,
8  * <URL: http://www.shoup.net/papers/oaep.ps.Z>
9  * for problems with the security proof for the
10  * original OAEP scheme, which EME-OAEP is based on.
11  *
12  * A new proof can be found in E. Fujisaki, T. Okamoto,
13  * D. Pointcheval, J. Stern, "RSA-OEAP is Still Alive!",
14  * Dec. 2000, <URL: http://eprint.iacr.org/2000/061/>.
15  * The new proof has stronger requirements for the
16  * underlying permutation: "partial-one-wayness" instead
17  * of one-wayness.  For the RSA function, this is
18  * an equivalent notion.
19  */
20 
21 #include <stdio.h>
22 #include <string.h>
23 
24 #if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1)
25 #include "cryptlib.h"
26 #include <openssl/bn.h>
27 #include <openssl/rsa.h>
28 #include <openssl/evp.h>
29 #include <openssl/rand.h>
30 #include <openssl/sha.h>
31 
32 static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
33     long seedlen);
34 
35 int
36 RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
37     const unsigned char *from, int flen, const unsigned char *param, int plen)
38 {
39 	int i, emlen = tlen - 1;
40 	unsigned char *db, *seed;
41 	unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];
42 
43 	if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1) {
44 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
45 		    RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
46 		return 0;
47 	}
48 
49 	if (emlen < 2 * SHA_DIGEST_LENGTH + 1) {
50 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
51 		    RSA_R_KEY_SIZE_TOO_SMALL);
52 		return 0;
53 	}
54 
55 	to[0] = 0;
56 	seed = to + 1;
57 	db = to + SHA_DIGEST_LENGTH + 1;
58 
59 	if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL))
60 		return 0;
61 	memset(db + SHA_DIGEST_LENGTH, 0,
62 	    emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
63 	db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
64 	memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, flen);
65 	if (RAND_bytes(seed, SHA_DIGEST_LENGTH) <= 0)
66 		return 0;
67 
68 	dbmask = malloc(emlen - SHA_DIGEST_LENGTH);
69 	if (dbmask == NULL) {
70 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
71 		return 0;
72 	}
73 
74 	if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed,
75 	    SHA_DIGEST_LENGTH) < 0)
76 		return 0;
77 	for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
78 		db[i] ^= dbmask[i];
79 
80 	if (MGF1(seedmask, SHA_DIGEST_LENGTH, db,
81 	    emlen - SHA_DIGEST_LENGTH) < 0)
82 		return 0;
83 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
84 		seed[i] ^= seedmask[i];
85 
86 	free(dbmask);
87 	return 1;
88 }
89 
90 int
91 RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
92     const unsigned char *from, int flen, int num, const unsigned char *param,
93     int plen)
94 {
95 	int i, dblen, mlen = -1;
96 	const unsigned char *maskeddb;
97 	int lzero;
98 	unsigned char *db = NULL;
99 	unsigned char seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
100 	unsigned char *padded_from;
101 	int bad = 0;
102 
103 	if (--num < 2 * SHA_DIGEST_LENGTH + 1)
104 		/*
105 		 * 'num' is the length of the modulus, i.e. does not depend
106 		 * on the particular ciphertext.
107 		 */
108 		goto decoding_err;
109 
110 	lzero = num - flen;
111 	if (lzero < 0) {
112 		/*
113 		 * signalling this error immediately after detection might allow
114 		 * for side-channel attacks (e.g. timing if 'plen' is huge
115 		 * -- cf. James H. Manger, "A Chosen Ciphertext Attack on RSA
116 		 * Optimal Asymmetric Encryption Padding (OAEP) [...]",
117 		 * CRYPTO 2001), so we use a 'bad' flag
118 		 */
119 		bad = 1;
120 		lzero = 0;
121 		flen = num; /* don't overflow the memcpy to padded_from */
122 	}
123 
124 	dblen = num - SHA_DIGEST_LENGTH;
125 	db = malloc(dblen + num);
126 	if (db == NULL) {
127 		RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
128 		    ERR_R_MALLOC_FAILURE);
129 		return -1;
130 	}
131 
132 	/*
133 	 * Always do this zero-padding copy (even when lzero == 0)
134 	 * to avoid leaking timing info about the value of lzero.
135 	 */
136 	padded_from = db + dblen;
137 	memset(padded_from, 0, lzero);
138 	memcpy(padded_from + lzero, from, flen);
139 
140 	maskeddb = padded_from + SHA_DIGEST_LENGTH;
141 
142 	if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
143 		return -1;
144 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
145 		seed[i] ^= padded_from[i];
146 
147 	if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
148 		return -1;
149 	for (i = 0; i < dblen; i++)
150 		db[i] ^= maskeddb[i];
151 
152 	if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL))
153 		return -1;
154 
155 	if (CRYPTO_memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
156 		goto decoding_err;
157 	else {
158 		for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
159 			if (db[i] != 0x00)
160 				break;
161 		if (i == dblen || db[i] != 0x01)
162 			goto decoding_err;
163 		else {
164 			/* everything looks OK */
165 
166 			mlen = dblen - ++i;
167 			if (tlen < mlen) {
168 				RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
169 				    RSA_R_DATA_TOO_LARGE);
170 				mlen = -1;
171 			} else
172 				memcpy(to, db + i, mlen);
173 		}
174 	}
175 	free(db);
176 	return mlen;
177 
178 decoding_err:
179 	/*
180 	 * To avoid chosen ciphertext attacks, the error message should not
181 	 * reveal which kind of decoding error happened
182 	 */
183 	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
184 	free(db);
185 	return -1;
186 }
187 
188 int
189 PKCS1_MGF1(unsigned char *mask, long len, const unsigned char *seed,
190     long seedlen, const EVP_MD *dgst)
191 {
192 	long i, outlen = 0;
193 	unsigned char cnt[4];
194 	EVP_MD_CTX c;
195 	unsigned char md[EVP_MAX_MD_SIZE];
196 	int mdlen;
197 	int rv = -1;
198 
199 	EVP_MD_CTX_init(&c);
200 	mdlen = EVP_MD_size(dgst);
201 	if (mdlen < 0)
202 		goto err;
203 	for (i = 0; outlen < len; i++) {
204 		cnt[0] = (unsigned char)((i >> 24) & 255);
205 		cnt[1] = (unsigned char)((i >> 16) & 255);
206 		cnt[2] = (unsigned char)((i >> 8)) & 255;
207 		cnt[3] = (unsigned char)(i & 255);
208 		if (!EVP_DigestInit_ex(&c, dgst, NULL) ||
209 		    !EVP_DigestUpdate(&c, seed, seedlen) ||
210 		    !EVP_DigestUpdate(&c, cnt, 4))
211 			goto err;
212 		if (outlen + mdlen <= len) {
213 			if (!EVP_DigestFinal_ex(&c, mask + outlen, NULL))
214 				goto err;
215 			outlen += mdlen;
216 		} else {
217 			if (!EVP_DigestFinal_ex(&c, md, NULL))
218 				goto err;
219 			memcpy(mask + outlen, md, len - outlen);
220 			outlen = len;
221 		}
222 	}
223 	rv = 0;
224 err:
225 	EVP_MD_CTX_cleanup(&c);
226 	return rv;
227 }
228 
229 static int
230 MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
231 {
232 	return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
233 }
234 #endif
235