xref: /openbsd-src/lib/libcrypto/rsa/rsa_oaep.c (revision 50b7afb2c2c0993b0894d4e34bf857cb13ed9c80)
1 /* $OpenBSD: rsa_oaep.c,v 1.23 2014/07/11 08:44:49 jsing Exp $ */
2 /* Written by Ulf Moeller. This software is distributed on an "AS IS"
3    basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. */
4 
5 /* EME-OAEP as defined in RFC 2437 (PKCS #1 v2.0) */
6 
7 /* See Victor Shoup, "OAEP reconsidered," Nov. 2000,
8  * <URL: http://www.shoup.net/papers/oaep.ps.Z>
9  * for problems with the security proof for the
10  * original OAEP scheme, which EME-OAEP is based on.
11  *
12  * A new proof can be found in E. Fujisaki, T. Okamoto,
13  * D. Pointcheval, J. Stern, "RSA-OEAP is Still Alive!",
14  * Dec. 2000, <URL: http://eprint.iacr.org/2000/061/>.
15  * The new proof has stronger requirements for the
16  * underlying permutation: "partial-one-wayness" instead
17  * of one-wayness.  For the RSA function, this is
18  * an equivalent notion.
19  */
20 
21 #include <stdio.h>
22 #include <string.h>
23 
24 #include <openssl/opensslconf.h>
25 
26 #if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1)
27 
28 #include <openssl/bn.h>
29 #include <openssl/err.h>
30 #include <openssl/evp.h>
31 #include <openssl/rand.h>
32 #include <openssl/rsa.h>
33 #include <openssl/sha.h>
34 
35 static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
36     long seedlen);
37 
38 int
39 RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
40     const unsigned char *from, int flen, const unsigned char *param, int plen)
41 {
42 	int i, emlen = tlen - 1;
43 	unsigned char *db, *seed;
44 	unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];
45 
46 	if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1) {
47 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
48 		    RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
49 		return 0;
50 	}
51 
52 	if (emlen < 2 * SHA_DIGEST_LENGTH + 1) {
53 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
54 		    RSA_R_KEY_SIZE_TOO_SMALL);
55 		return 0;
56 	}
57 
58 	to[0] = 0;
59 	seed = to + 1;
60 	db = to + SHA_DIGEST_LENGTH + 1;
61 
62 	if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL))
63 		return 0;
64 	memset(db + SHA_DIGEST_LENGTH, 0,
65 	    emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
66 	db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
67 	memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, flen);
68 	if (RAND_bytes(seed, SHA_DIGEST_LENGTH) <= 0)
69 		return 0;
70 
71 	dbmask = malloc(emlen - SHA_DIGEST_LENGTH);
72 	if (dbmask == NULL) {
73 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
74 		return 0;
75 	}
76 
77 	if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed,
78 	    SHA_DIGEST_LENGTH) < 0)
79 		return 0;
80 	for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
81 		db[i] ^= dbmask[i];
82 
83 	if (MGF1(seedmask, SHA_DIGEST_LENGTH, db,
84 	    emlen - SHA_DIGEST_LENGTH) < 0)
85 		return 0;
86 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
87 		seed[i] ^= seedmask[i];
88 
89 	free(dbmask);
90 	return 1;
91 }
92 
93 int
94 RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
95     const unsigned char *from, int flen, int num, const unsigned char *param,
96     int plen)
97 {
98 	int i, dblen, mlen = -1;
99 	const unsigned char *maskeddb;
100 	int lzero;
101 	unsigned char *db = NULL;
102 	unsigned char seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
103 	unsigned char *padded_from;
104 	int bad = 0;
105 
106 	if (--num < 2 * SHA_DIGEST_LENGTH + 1)
107 		/*
108 		 * 'num' is the length of the modulus, i.e. does not depend
109 		 * on the particular ciphertext.
110 		 */
111 		goto decoding_err;
112 
113 	lzero = num - flen;
114 	if (lzero < 0) {
115 		/*
116 		 * signalling this error immediately after detection might allow
117 		 * for side-channel attacks (e.g. timing if 'plen' is huge
118 		 * -- cf. James H. Manger, "A Chosen Ciphertext Attack on RSA
119 		 * Optimal Asymmetric Encryption Padding (OAEP) [...]",
120 		 * CRYPTO 2001), so we use a 'bad' flag
121 		 */
122 		bad = 1;
123 		lzero = 0;
124 		flen = num; /* don't overflow the memcpy to padded_from */
125 	}
126 
127 	dblen = num - SHA_DIGEST_LENGTH;
128 	db = malloc(dblen + num);
129 	if (db == NULL) {
130 		RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
131 		    ERR_R_MALLOC_FAILURE);
132 		return -1;
133 	}
134 
135 	/*
136 	 * Always do this zero-padding copy (even when lzero == 0)
137 	 * to avoid leaking timing info about the value of lzero.
138 	 */
139 	padded_from = db + dblen;
140 	memset(padded_from, 0, lzero);
141 	memcpy(padded_from + lzero, from, flen);
142 
143 	maskeddb = padded_from + SHA_DIGEST_LENGTH;
144 
145 	if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
146 		return -1;
147 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
148 		seed[i] ^= padded_from[i];
149 
150 	if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
151 		return -1;
152 	for (i = 0; i < dblen; i++)
153 		db[i] ^= maskeddb[i];
154 
155 	if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL))
156 		return -1;
157 
158 	if (CRYPTO_memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
159 		goto decoding_err;
160 	else {
161 		for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
162 			if (db[i] != 0x00)
163 				break;
164 		if (i == dblen || db[i] != 0x01)
165 			goto decoding_err;
166 		else {
167 			/* everything looks OK */
168 
169 			mlen = dblen - ++i;
170 			if (tlen < mlen) {
171 				RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
172 				    RSA_R_DATA_TOO_LARGE);
173 				mlen = -1;
174 			} else
175 				memcpy(to, db + i, mlen);
176 		}
177 	}
178 	free(db);
179 	return mlen;
180 
181 decoding_err:
182 	/*
183 	 * To avoid chosen ciphertext attacks, the error message should not
184 	 * reveal which kind of decoding error happened
185 	 */
186 	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
187 	free(db);
188 	return -1;
189 }
190 
191 int
192 PKCS1_MGF1(unsigned char *mask, long len, const unsigned char *seed,
193     long seedlen, const EVP_MD *dgst)
194 {
195 	long i, outlen = 0;
196 	unsigned char cnt[4];
197 	EVP_MD_CTX c;
198 	unsigned char md[EVP_MAX_MD_SIZE];
199 	int mdlen;
200 	int rv = -1;
201 
202 	EVP_MD_CTX_init(&c);
203 	mdlen = EVP_MD_size(dgst);
204 	if (mdlen < 0)
205 		goto err;
206 	for (i = 0; outlen < len; i++) {
207 		cnt[0] = (unsigned char)((i >> 24) & 255);
208 		cnt[1] = (unsigned char)((i >> 16) & 255);
209 		cnt[2] = (unsigned char)((i >> 8)) & 255;
210 		cnt[3] = (unsigned char)(i & 255);
211 		if (!EVP_DigestInit_ex(&c, dgst, NULL) ||
212 		    !EVP_DigestUpdate(&c, seed, seedlen) ||
213 		    !EVP_DigestUpdate(&c, cnt, 4))
214 			goto err;
215 		if (outlen + mdlen <= len) {
216 			if (!EVP_DigestFinal_ex(&c, mask + outlen, NULL))
217 				goto err;
218 			outlen += mdlen;
219 		} else {
220 			if (!EVP_DigestFinal_ex(&c, md, NULL))
221 				goto err;
222 			memcpy(mask + outlen, md, len - outlen);
223 			outlen = len;
224 		}
225 	}
226 	rv = 0;
227 err:
228 	EVP_MD_CTX_cleanup(&c);
229 	return rv;
230 }
231 
232 static int
233 MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
234 {
235 	return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
236 }
237 #endif
238