xref: /openbsd-src/lib/libcrypto/rsa/rsa_pk1.c (revision 3516dac2040b81c02de01ace3844ac6ee9959540)
1 /* $OpenBSD: rsa_pk1.c,v 1.17 2024/03/30 04:34:17 jsing Exp $ */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  *
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  *
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  *
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  *
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58 
59 #include <stdio.h>
60 #include <stdlib.h>
61 #include <string.h>
62 
63 #include <openssl/bn.h>
64 #include <openssl/err.h>
65 #include <openssl/rsa.h>
66 
67 #include "bytestring.h"
68 
69 int
RSA_padding_add_PKCS1_type_1(unsigned char * to,int tlen,const unsigned char * from,int flen)70 RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen,
71     const unsigned char *from, int flen)
72 {
73 	CBB cbb;
74 	int i;
75 	int ret = 0;
76 
77 	/*
78 	 * Pad data block with PKCS1 type 1 padding - RFC 2313, section 8.1.
79 	 */
80 
81 	memset(&cbb, 0, sizeof(cbb));
82 
83 	if (flen < 0 || tlen < 0)
84 		goto err;
85 
86 	if (flen > tlen - RSA_PKCS1_PADDING_SIZE) {
87 		RSAerror(RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
88 		goto err;
89 	}
90 
91 	if (!CBB_init_fixed(&cbb, to, tlen))
92 		goto err;
93 
94 	/*
95 	 * Add leading NUL, block type (0x01), padding bytes (0xff) and
96 	 * trailing NUL.
97 	 */
98 	if (!CBB_add_u8(&cbb, 0))
99 		goto err;
100 	if (!CBB_add_u8(&cbb, 1))
101 		goto err;
102 	for (i = 0; i < tlen - 3 - flen; i++) {
103 		if (!CBB_add_u8(&cbb, 0xff))
104 			goto err;
105 	}
106 	if (!CBB_add_u8(&cbb, 0))
107 		goto err;
108 
109 	/* Now add the actual data. */
110 	if (!CBB_add_bytes(&cbb, from, flen))
111 		goto err;
112 
113 	if (!CBB_finish(&cbb, NULL, NULL))
114 		goto err;
115 
116 	ret = 1;
117 
118  err:
119 	CBB_cleanup(&cbb);
120 
121 	return ret;
122 }
123 LCRYPTO_ALIAS(RSA_padding_add_PKCS1_type_1);
124 
125 int
RSA_padding_check_PKCS1_type_1(unsigned char * to,int tlen,const unsigned char * from,int flen,int num)126 RSA_padding_check_PKCS1_type_1(unsigned char *to, int tlen,
127     const unsigned char *from, int flen, int num)
128 {
129 	int i, j;
130 	const unsigned char *p;
131 
132 	p = from;
133 	if (num != flen + 1 || *(p++) != 01) {
134 		RSAerror(RSA_R_BLOCK_TYPE_IS_NOT_01);
135 		return -1;
136 	}
137 
138 	/* scan over padding data */
139 	j = flen - 1; /* one for type. */
140 	for (i = 0; i < j; i++) {
141 		if (*p != 0xff) {
142 			/* should decrypt to 0xff */
143 			if (*p == 0) {
144 				p++;
145 				break;
146 			} else {
147 				RSAerror(RSA_R_BAD_FIXED_HEADER_DECRYPT);
148 				return -1;
149 			}
150 		}
151 		p++;
152 	}
153 
154 	if (i == j) {
155 		RSAerror(RSA_R_NULL_BEFORE_BLOCK_MISSING);
156 		return -1;
157 	}
158 
159 	if (i < 8) {
160 		RSAerror(RSA_R_BAD_PAD_BYTE_COUNT);
161 		return -1;
162 	}
163 	i++; /* Skip over the '\0' */
164 	j -= i;
165 	if (j > tlen) {
166 		RSAerror(RSA_R_DATA_TOO_LARGE);
167 		return -1;
168 	}
169 	memcpy(to, p, j);
170 
171 	return j;
172 }
173 LCRYPTO_ALIAS(RSA_padding_check_PKCS1_type_1);
174 
175 int
RSA_padding_add_PKCS1_type_2(unsigned char * to,int tlen,const unsigned char * from,int flen)176 RSA_padding_add_PKCS1_type_2(unsigned char *to, int tlen,
177     const unsigned char *from, int flen)
178 {
179 	uint8_t padding[256];
180 	uint8_t pad;
181 	CBB cbb;
182 	CBS cbs;
183 	int i;
184 	int ret = 0;
185 
186 	/*
187 	 * Pad data block with PKCS1 type 2 padding - RFC 2313, section 8.1.
188 	 */
189 
190 	memset(&cbb, 0, sizeof(cbb));
191 	CBS_init(&cbs, NULL, 0);
192 
193 	if (flen < 0 || tlen < 0)
194 		goto err;
195 
196 	if (flen > tlen - RSA_PKCS1_PADDING_SIZE) {
197 		RSAerror(RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
198 		goto err;
199 	}
200 
201 	if (!CBB_init_fixed(&cbb, to, tlen))
202 		goto err;
203 
204 	/*
205 	 * Add leading NUL, block type (0x02), padding bytes (random non-zero
206 	 * bytes) and trailing NUL.
207 	 */
208 	if (!CBB_add_u8(&cbb, 0))
209 		goto err;
210 	if (!CBB_add_u8(&cbb, 2))
211 		goto err;
212 	for (i = 0; i < tlen - 3 - flen; i++) {
213 		do {
214 			if (CBS_len(&cbs) == 0) {
215 				arc4random_buf(padding, sizeof(padding));
216 				CBS_init(&cbs, padding, sizeof(padding));
217 			}
218 			if (!CBS_get_u8(&cbs, &pad))
219 				goto err;
220 		} while (pad == 0);
221 
222 		if (!CBB_add_u8(&cbb, pad))
223 			goto err;
224 	}
225 	if (!CBB_add_u8(&cbb, 0))
226 		goto err;
227 
228 	/* Now add the actual data. */
229 	if (!CBB_add_bytes(&cbb, from, flen))
230 		goto err;
231 
232 	if (!CBB_finish(&cbb, NULL, NULL))
233 		goto err;
234 
235 	ret = 1;
236 
237  err:
238 	CBB_cleanup(&cbb);
239 	explicit_bzero(padding, sizeof(padding));
240 
241 	return ret;
242 }
243 LCRYPTO_ALIAS(RSA_padding_add_PKCS1_type_2);
244 
245 int
RSA_padding_check_PKCS1_type_2(unsigned char * to,int tlen,const unsigned char * from,int flen,int num)246 RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen,
247     const unsigned char *from, int flen, int num)
248 {
249 	int i, j;
250 	const unsigned char *p;
251 
252 	p = from;
253 	if (num != flen + 1 || *(p++) != 02) {
254 		RSAerror(RSA_R_BLOCK_TYPE_IS_NOT_02);
255 		return -1;
256 	}
257 
258 	/* scan over padding data */
259 	j = flen - 1; /* one for type. */
260 	for (i = 0; i < j; i++)
261 		if (*(p++) == 0)
262 			break;
263 
264 	if (i == j) {
265 		RSAerror(RSA_R_NULL_BEFORE_BLOCK_MISSING);
266 		return -1;
267 	}
268 
269 	if (i < 8) {
270 		RSAerror(RSA_R_BAD_PAD_BYTE_COUNT);
271 		return -1;
272 	}
273 	i++; /* Skip over the '\0' */
274 	j -= i;
275 	if (j > tlen) {
276 		RSAerror(RSA_R_DATA_TOO_LARGE);
277 		return -1;
278 	}
279 	memcpy(to, p, j);
280 
281 	return j;
282 }
283 LCRYPTO_ALIAS(RSA_padding_check_PKCS1_type_2);
284