xref: /openbsd-src/regress/lib/libcrypto/rsa/rsa_padding_test.c (revision e10c7425818047f2f5dbec51b5086d21c1b11636)

/*	$OpenBSD: rsa_padding_test.c,v 1.2 2024/03/30 02:20:39 jsing Exp $	*/
/*
 * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <stdint.h>
#include <string.h>

#include <openssl/err.h>
#include <openssl/rsa.h>

#if 0
static void
hexdump(const unsigned char *buf, size_t len)
{
	size_t i;

	for (i = 1; i <= len; i++)
		fprintf(stderr, " 0x%02hhx,%s", buf[i - 1], i % 8 ? "" : "\n");

	fprintf(stderr, "\n");
}
#endif

struct pkcs1_test {
	uint8_t in[128];
	size_t in_len;
	int want;
	int want_error;
};

static const struct pkcs1_test pkcs1_type1_tests[] = {
	{
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
			0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
			0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
		},
		.in_len = 32,
		.want = 19,
	},
	{
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00,
		},
		.in_len = 11,
		.want = 0,
	},
	{
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00, 0xff,
		},
		.in_len = 12,
		.want = 1,
	},
	{
		/* Insufficient padding bytes (< 8). */
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0x00, 0xff, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BAD_PAD_BYTE_COUNT,
	},
	{
		/* Incorrect padding type (0x00). */
		.in = {
			0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
	},
	{
		/* Incorrect padding type (0x02). */
		.in = {
			0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
	},
	{
		/* Non-padding byte before end of padding marker. */
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xfe, 0x00, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BAD_FIXED_HEADER_DECRYPT,
	},
	{
		/* No end of padding marker. */
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
		},
		.in_len = 32,
		.want = -1,
		.want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
	},
};

#define N_PKCS1_TYPE1_TESTS \
	(sizeof(pkcs1_type1_tests) / sizeof(pkcs1_type1_tests[0]))

static int
test_pkcs1_type1(void)
{
	const struct pkcs1_test *pt;
	uint8_t buf[32], in[19], out[512];
	int pad_len;
	long err;
	size_t i;
	int failed = 1;

	for (i = 0; i < 1000; i++) {
		arc4random_buf(in, sizeof(in));

		if (!RSA_padding_add_PKCS1_type_1(buf, sizeof(buf), in,
		    sizeof(in))) {
			fprintf(stderr, "FAIL: failed to add PKCS1 type 1 "
			    "padding\n");
			goto failed;
		}

		pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
		    buf + 1, sizeof(buf) - 1, sizeof(buf));
		if (pad_len != sizeof(in)) {
			fprintf(stderr, "FAIL: failed to check PKCS1 type 1 "
			    "padding\n");
			ERR_print_errors_fp(stderr);
			goto failed;
		}
	}

	for (i = 0; i < N_PKCS1_TYPE1_TESTS; i++) {
		pt = &pkcs1_type1_tests[i];

		ERR_clear_error();

		pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
		    pt->in + 1, pt->in_len - 1, pt->in_len);

		if (pad_len != pt->want) {
			fprintf(stderr, "FAIL: test %zu - failed to check "
			    "PKCS1 type 1 padding (%d != %d)\n", i, pad_len,
			    pt->want);
			ERR_print_errors_fp(stderr);
			goto failed;
		}

		err = ERR_peek_error();
		if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
			fprintf(stderr, "FAIL: test %zu - PKCS1 type 1 padding "
			    "check failed with error reason %i, want %i\n",
			    i, ERR_GET_REASON(err), pt->want_error);
			ERR_print_errors_fp(stderr);
			goto failed;
		}
	}

	failed = 0;

 failed:
	return failed;
}

static const struct pkcs1_test pkcs1_type2_tests[] = {
	{
		.in = {
			0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
			0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
			0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
		},
		.in_len = 32,
		.want = 19,
	},
	{
		.in = {
			0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00,
		},
		.in_len = 11,
		.want = 0,
	},
	{
		.in = {
			0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00, 0xff,
		},
		.in_len = 12,
		.want = 1,
	},
	{
		/* Insufficient padding bytes (< 8). */
		.in = {
			0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0x00, 0xff, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BAD_PAD_BYTE_COUNT,
	},
	{
		/* Incorrect padding type (0x00). */
		.in = {
			0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
	},
	{
		/* Incorrect padding type (0x01). */
		.in = {
			0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0x00, 0xff,
		},
		.in_len = 12,
		.want = -1,
		.want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
	},
	{
		/* No end of padding marker. */
		.in = {
			0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			0xff, 0xff, 0xff, 0xff, 0xff, 0x6f, 0x6f, 0x6f,
			0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
			0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
		},
		.in_len = 32,
		.want = -1,
		.want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
	},
};

#define N_PKCS1_TYPE2_TESTS \
	(sizeof(pkcs1_type2_tests) / sizeof(pkcs1_type2_tests[0]))

static int
test_pkcs1_type2(void)
{
	const struct pkcs1_test *pt;
	uint8_t buf[32], in[19], out[512];
	int pad_len;
	long err;
	size_t i;
	int failed = 1;

	for (i = 0; i < 1000; i++) {
		arc4random_buf(in, sizeof(in));

		if (!RSA_padding_add_PKCS1_type_2(buf, sizeof(buf), in,
		    sizeof(in))) {
			fprintf(stderr, "FAIL: failed to add PKCS1 type 2 "
			    "padding\n");
			goto failed;
		}

		pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
		    buf + 1, sizeof(buf) - 1, sizeof(buf));
		if (pad_len != sizeof(in)) {
			fprintf(stderr, "FAIL: failed to check PKCS1 type 2 "
			    "padding\n");
			ERR_print_errors_fp(stderr);
			goto failed;
		}
	}

	for (i = 0; i < N_PKCS1_TYPE2_TESTS; i++) {
		pt = &pkcs1_type2_tests[i];

		ERR_clear_error();

		pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
		    pt->in + 1, pt->in_len - 1, pt->in_len);

		if (pad_len != pt->want) {
			fprintf(stderr, "FAIL: test %zu - failed to check "
			    "PKCS1 type 2 padding (%d != %d)\n", i, pad_len,
			    pt->want);
			ERR_print_errors_fp(stderr);
			goto failed;
		}

		err = ERR_peek_error();
		if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
			fprintf(stderr, "FAIL: test %zu - PKCS1 type 2 padding "
			    "check failed with error reason %i, want %i\n",
			    i, ERR_GET_REASON(err), pt->want_error);
			ERR_print_errors_fp(stderr);
			goto failed;
		}
	}

	failed = 0;

 failed:
	return failed;
}

int
main(int argc, char **argv)
{
	int failed = 0;

	failed |= test_pkcs1_type1();
	failed |= test_pkcs1_type2();

	return failed;
}