1 /* $OpenBSD: rsa_padding_test.c,v 1.2 2024/03/30 02:20:39 jsing Exp $ */
2 /*
3 * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18 #include <stdint.h>
19 #include <string.h>
20
21 #include <openssl/err.h>
22 #include <openssl/rsa.h>
23
24 #if 0
25 static void
26 hexdump(const unsigned char *buf, size_t len)
27 {
28 size_t i;
29
30 for (i = 1; i <= len; i++)
31 fprintf(stderr, " 0x%02hhx,%s", buf[i - 1], i % 8 ? "" : "\n");
32
33 fprintf(stderr, "\n");
34 }
35 #endif
36
37 struct pkcs1_test {
38 uint8_t in[128];
39 size_t in_len;
40 int want;
41 int want_error;
42 };
43
44 static const struct pkcs1_test pkcs1_type1_tests[] = {
45 {
46 .in = {
47 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
48 0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
49 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
50 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
51 },
52 .in_len = 32,
53 .want = 19,
54 },
55 {
56 .in = {
57 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
58 0xff, 0xff, 0x00,
59 },
60 .in_len = 11,
61 .want = 0,
62 },
63 {
64 .in = {
65 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
66 0xff, 0xff, 0x00, 0xff,
67 },
68 .in_len = 12,
69 .want = 1,
70 },
71 {
72 /* Insufficient padding bytes (< 8). */
73 .in = {
74 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
75 0xff, 0x00, 0xff, 0xff,
76 },
77 .in_len = 12,
78 .want = -1,
79 .want_error = RSA_R_BAD_PAD_BYTE_COUNT,
80 },
81 {
82 /* Incorrect padding type (0x00). */
83 .in = {
84 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
85 0xff, 0xff, 0x00, 0xff,
86 },
87 .in_len = 12,
88 .want = -1,
89 .want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
90 },
91 {
92 /* Incorrect padding type (0x02). */
93 .in = {
94 0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
95 0xff, 0xff, 0x00, 0xff,
96 },
97 .in_len = 12,
98 .want = -1,
99 .want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
100 },
101 {
102 /* Non-padding byte before end of padding marker. */
103 .in = {
104 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
105 0xff, 0xfe, 0x00, 0xff,
106 },
107 .in_len = 12,
108 .want = -1,
109 .want_error = RSA_R_BAD_FIXED_HEADER_DECRYPT,
110 },
111 {
112 /* No end of padding marker. */
113 .in = {
114 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
115 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
116 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
117 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
118 },
119 .in_len = 32,
120 .want = -1,
121 .want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
122 },
123 };
124
125 #define N_PKCS1_TYPE1_TESTS \
126 (sizeof(pkcs1_type1_tests) / sizeof(pkcs1_type1_tests[0]))
127
128 static int
test_pkcs1_type1(void)129 test_pkcs1_type1(void)
130 {
131 const struct pkcs1_test *pt;
132 uint8_t buf[32], in[19], out[512];
133 int pad_len;
134 long err;
135 size_t i;
136 int failed = 1;
137
138 for (i = 0; i < 1000; i++) {
139 arc4random_buf(in, sizeof(in));
140
141 if (!RSA_padding_add_PKCS1_type_1(buf, sizeof(buf), in,
142 sizeof(in))) {
143 fprintf(stderr, "FAIL: failed to add PKCS1 type 1 "
144 "padding\n");
145 goto failed;
146 }
147
148 pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
149 buf + 1, sizeof(buf) - 1, sizeof(buf));
150 if (pad_len != sizeof(in)) {
151 fprintf(stderr, "FAIL: failed to check PKCS1 type 1 "
152 "padding\n");
153 ERR_print_errors_fp(stderr);
154 goto failed;
155 }
156 }
157
158 for (i = 0; i < N_PKCS1_TYPE1_TESTS; i++) {
159 pt = &pkcs1_type1_tests[i];
160
161 ERR_clear_error();
162
163 pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
164 pt->in + 1, pt->in_len - 1, pt->in_len);
165
166 if (pad_len != pt->want) {
167 fprintf(stderr, "FAIL: test %zu - failed to check "
168 "PKCS1 type 1 padding (%d != %d)\n", i, pad_len,
169 pt->want);
170 ERR_print_errors_fp(stderr);
171 goto failed;
172 }
173
174 err = ERR_peek_error();
175 if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
176 fprintf(stderr, "FAIL: test %zu - PKCS1 type 1 padding "
177 "check failed with error reason %i, want %i\n",
178 i, ERR_GET_REASON(err), pt->want_error);
179 ERR_print_errors_fp(stderr);
180 goto failed;
181 }
182 }
183
184 failed = 0;
185
186 failed:
187 return failed;
188 }
189
190 static const struct pkcs1_test pkcs1_type2_tests[] = {
191 {
192 .in = {
193 0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
194 0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
195 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
196 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
197 },
198 .in_len = 32,
199 .want = 19,
200 },
201 {
202 .in = {
203 0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
204 0xff, 0xff, 0x00,
205 },
206 .in_len = 11,
207 .want = 0,
208 },
209 {
210 .in = {
211 0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
212 0xff, 0xff, 0x00, 0xff,
213 },
214 .in_len = 12,
215 .want = 1,
216 },
217 {
218 /* Insufficient padding bytes (< 8). */
219 .in = {
220 0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
221 0xff, 0x00, 0xff, 0xff,
222 },
223 .in_len = 12,
224 .want = -1,
225 .want_error = RSA_R_BAD_PAD_BYTE_COUNT,
226 },
227 {
228 /* Incorrect padding type (0x00). */
229 .in = {
230 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
231 0xff, 0xff, 0x00, 0xff,
232 },
233 .in_len = 12,
234 .want = -1,
235 .want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
236 },
237 {
238 /* Incorrect padding type (0x01). */
239 .in = {
240 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
241 0xff, 0xff, 0x00, 0xff,
242 },
243 .in_len = 12,
244 .want = -1,
245 .want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
246 },
247 {
248 /* No end of padding marker. */
249 .in = {
250 0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
251 0xff, 0xff, 0xff, 0xff, 0xff, 0x6f, 0x6f, 0x6f,
252 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
253 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
254 },
255 .in_len = 32,
256 .want = -1,
257 .want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
258 },
259 };
260
261 #define N_PKCS1_TYPE2_TESTS \
262 (sizeof(pkcs1_type2_tests) / sizeof(pkcs1_type2_tests[0]))
263
264 static int
test_pkcs1_type2(void)265 test_pkcs1_type2(void)
266 {
267 const struct pkcs1_test *pt;
268 uint8_t buf[32], in[19], out[512];
269 int pad_len;
270 long err;
271 size_t i;
272 int failed = 1;
273
274 for (i = 0; i < 1000; i++) {
275 arc4random_buf(in, sizeof(in));
276
277 if (!RSA_padding_add_PKCS1_type_2(buf, sizeof(buf), in,
278 sizeof(in))) {
279 fprintf(stderr, "FAIL: failed to add PKCS1 type 2 "
280 "padding\n");
281 goto failed;
282 }
283
284 pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
285 buf + 1, sizeof(buf) - 1, sizeof(buf));
286 if (pad_len != sizeof(in)) {
287 fprintf(stderr, "FAIL: failed to check PKCS1 type 2 "
288 "padding\n");
289 ERR_print_errors_fp(stderr);
290 goto failed;
291 }
292 }
293
294 for (i = 0; i < N_PKCS1_TYPE2_TESTS; i++) {
295 pt = &pkcs1_type2_tests[i];
296
297 ERR_clear_error();
298
299 pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
300 pt->in + 1, pt->in_len - 1, pt->in_len);
301
302 if (pad_len != pt->want) {
303 fprintf(stderr, "FAIL: test %zu - failed to check "
304 "PKCS1 type 2 padding (%d != %d)\n", i, pad_len,
305 pt->want);
306 ERR_print_errors_fp(stderr);
307 goto failed;
308 }
309
310 err = ERR_peek_error();
311 if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
312 fprintf(stderr, "FAIL: test %zu - PKCS1 type 2 padding "
313 "check failed with error reason %i, want %i\n",
314 i, ERR_GET_REASON(err), pt->want_error);
315 ERR_print_errors_fp(stderr);
316 goto failed;
317 }
318 }
319
320 failed = 0;
321
322 failed:
323 return failed;
324 }
325
326 int
main(int argc,char ** argv)327 main(int argc, char **argv)
328 {
329 int failed = 0;
330
331 failed |= test_pkcs1_type1();
332 failed |= test_pkcs1_type2();
333
334 return failed;
335 }
336