xref: /openbsd-src/lib/libcrypto/rsa/rsa_oaep.c (revision 897fc685943471cf985a0fe38ba076ea6fe74fa5)
1 /* $OpenBSD: rsa_oaep.c,v 1.26 2017/01/29 17:49:23 beck Exp $ */
2 /* Written by Ulf Moeller. This software is distributed on an "AS IS"
3    basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. */
4 
5 /* EME-OAEP as defined in RFC 2437 (PKCS #1 v2.0) */
6 
7 /* See Victor Shoup, "OAEP reconsidered," Nov. 2000,
8  * <URL: http://www.shoup.net/papers/oaep.ps.Z>
9  * for problems with the security proof for the
10  * original OAEP scheme, which EME-OAEP is based on.
11  *
12  * A new proof can be found in E. Fujisaki, T. Okamoto,
13  * D. Pointcheval, J. Stern, "RSA-OEAP is Still Alive!",
14  * Dec. 2000, <URL: http://eprint.iacr.org/2000/061/>.
15  * The new proof has stronger requirements for the
16  * underlying permutation: "partial-one-wayness" instead
17  * of one-wayness.  For the RSA function, this is
18  * an equivalent notion.
19  */
20 
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 
25 #include <openssl/opensslconf.h>
26 
27 #if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1)
28 
29 #include <openssl/bn.h>
30 #include <openssl/err.h>
31 #include <openssl/evp.h>
32 #include <openssl/rsa.h>
33 #include <openssl/sha.h>
34 
35 static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
36     long seedlen);
37 
38 int
39 RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
40     const unsigned char *from, int flen, const unsigned char *param, int plen)
41 {
42 	int i, emlen = tlen - 1;
43 	unsigned char *db, *seed;
44 	unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];
45 
46 	if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1) {
47 		RSAerror(RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
48 		return 0;
49 	}
50 
51 	if (emlen < 2 * SHA_DIGEST_LENGTH + 1) {
52 		RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
53 		return 0;
54 	}
55 
56 	to[0] = 0;
57 	seed = to + 1;
58 	db = to + SHA_DIGEST_LENGTH + 1;
59 
60 	if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL))
61 		return 0;
62 	memset(db + SHA_DIGEST_LENGTH, 0,
63 	    emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
64 	db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
65 	memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, flen);
66 	arc4random_buf(seed, SHA_DIGEST_LENGTH);
67 
68 	dbmask = malloc(emlen - SHA_DIGEST_LENGTH);
69 	if (dbmask == NULL) {
70 		RSAerror(ERR_R_MALLOC_FAILURE);
71 		return 0;
72 	}
73 
74 	if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed,
75 	    SHA_DIGEST_LENGTH) < 0)
76 		return 0;
77 	for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
78 		db[i] ^= dbmask[i];
79 
80 	if (MGF1(seedmask, SHA_DIGEST_LENGTH, db,
81 	    emlen - SHA_DIGEST_LENGTH) < 0)
82 		return 0;
83 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
84 		seed[i] ^= seedmask[i];
85 
86 	free(dbmask);
87 	return 1;
88 }
89 
90 int
91 RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
92     const unsigned char *from, int flen, int num, const unsigned char *param,
93     int plen)
94 {
95 	int i, dblen, mlen = -1;
96 	const unsigned char *maskeddb;
97 	int lzero;
98 	unsigned char *db = NULL;
99 	unsigned char seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
100 	unsigned char *padded_from;
101 	int bad = 0;
102 
103 	if (--num < 2 * SHA_DIGEST_LENGTH + 1)
104 		/*
105 		 * 'num' is the length of the modulus, i.e. does not depend
106 		 * on the particular ciphertext.
107 		 */
108 		goto decoding_err;
109 
110 	lzero = num - flen;
111 	if (lzero < 0) {
112 		/*
113 		 * signalling this error immediately after detection might allow
114 		 * for side-channel attacks (e.g. timing if 'plen' is huge
115 		 * -- cf. James H. Manger, "A Chosen Ciphertext Attack on RSA
116 		 * Optimal Asymmetric Encryption Padding (OAEP) [...]",
117 		 * CRYPTO 2001), so we use a 'bad' flag
118 		 */
119 		bad = 1;
120 		lzero = 0;
121 		flen = num; /* don't overflow the memcpy to padded_from */
122 	}
123 
124 	dblen = num - SHA_DIGEST_LENGTH;
125 	db = malloc(dblen + num);
126 	if (db == NULL) {
127 		RSAerror(ERR_R_MALLOC_FAILURE);
128 		return -1;
129 	}
130 
131 	/*
132 	 * Always do this zero-padding copy (even when lzero == 0)
133 	 * to avoid leaking timing info about the value of lzero.
134 	 */
135 	padded_from = db + dblen;
136 	memset(padded_from, 0, lzero);
137 	memcpy(padded_from + lzero, from, flen);
138 
139 	maskeddb = padded_from + SHA_DIGEST_LENGTH;
140 
141 	if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
142 		return -1;
143 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
144 		seed[i] ^= padded_from[i];
145 
146 	if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
147 		return -1;
148 	for (i = 0; i < dblen; i++)
149 		db[i] ^= maskeddb[i];
150 
151 	if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL))
152 		return -1;
153 
154 	if (timingsafe_memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
155 		goto decoding_err;
156 	else {
157 		for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
158 			if (db[i] != 0x00)
159 				break;
160 		if (i == dblen || db[i] != 0x01)
161 			goto decoding_err;
162 		else {
163 			/* everything looks OK */
164 
165 			mlen = dblen - ++i;
166 			if (tlen < mlen) {
167 				RSAerror(RSA_R_DATA_TOO_LARGE);
168 				mlen = -1;
169 			} else
170 				memcpy(to, db + i, mlen);
171 		}
172 	}
173 	free(db);
174 	return mlen;
175 
176 decoding_err:
177 	/*
178 	 * To avoid chosen ciphertext attacks, the error message should not
179 	 * reveal which kind of decoding error happened
180 	 */
181 	RSAerror(RSA_R_OAEP_DECODING_ERROR);
182 	free(db);
183 	return -1;
184 }
185 
186 int
187 PKCS1_MGF1(unsigned char *mask, long len, const unsigned char *seed,
188     long seedlen, const EVP_MD *dgst)
189 {
190 	long i, outlen = 0;
191 	unsigned char cnt[4];
192 	EVP_MD_CTX c;
193 	unsigned char md[EVP_MAX_MD_SIZE];
194 	int mdlen;
195 	int rv = -1;
196 
197 	EVP_MD_CTX_init(&c);
198 	mdlen = EVP_MD_size(dgst);
199 	if (mdlen < 0)
200 		goto err;
201 	for (i = 0; outlen < len; i++) {
202 		cnt[0] = (unsigned char)((i >> 24) & 255);
203 		cnt[1] = (unsigned char)((i >> 16) & 255);
204 		cnt[2] = (unsigned char)((i >> 8)) & 255;
205 		cnt[3] = (unsigned char)(i & 255);
206 		if (!EVP_DigestInit_ex(&c, dgst, NULL) ||
207 		    !EVP_DigestUpdate(&c, seed, seedlen) ||
208 		    !EVP_DigestUpdate(&c, cnt, 4))
209 			goto err;
210 		if (outlen + mdlen <= len) {
211 			if (!EVP_DigestFinal_ex(&c, mask + outlen, NULL))
212 				goto err;
213 			outlen += mdlen;
214 		} else {
215 			if (!EVP_DigestFinal_ex(&c, md, NULL))
216 				goto err;
217 			memcpy(mask + outlen, md, len - outlen);
218 			outlen = len;
219 		}
220 	}
221 	rv = 0;
222 err:
223 	EVP_MD_CTX_cleanup(&c);
224 	return rv;
225 }
226 
227 static int
228 MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
229 {
230 	return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
231 }
232 #endif
233