xref: /openbsd-src/lib/libcrypto/rsa/rsa_oaep.c (revision 913ec974266f8a62ab2e5ca34a31d6e6f75b3cf0)
1 /* crypto/rsa/rsa_oaep.c */
2 /* Written by Ulf Moeller. This software is distributed on an "AS IS"
3    basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. */
4 
5 /* EME_OAEP as defined in RFC 2437 (PKCS #1 v2.0) */
6 
7 #if !defined(NO_SHA) && !defined(NO_SHA1)
8 #include <stdio.h>
9 #include "cryptlib.h"
10 #include <openssl/bn.h>
11 #include <openssl/rsa.h>
12 #include <openssl/sha.h>
13 #include <openssl/rand.h>
14 
15 int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen);
16 
17 int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
18 	     unsigned char *from, int flen, unsigned char *param, int plen)
19     {
20     int i, emlen = tlen - 1;
21     unsigned char *db, *seed;
22     unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];
23 
24     if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1)
25 	{
26 	RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
27 	       RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
28 	return (0);
29 	}
30 
31     if (emlen < 2 * SHA_DIGEST_LENGTH + 1)
32 	{
33 	RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_KEY_SIZE_TOO_SMALL);
34 	return (0);
35 	}
36 
37     dbmask = Malloc(emlen - SHA_DIGEST_LENGTH);
38     if (dbmask == NULL)
39 	{
40 	RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
41 	return (0);
42 	}
43 
44     to[0] = 0;
45     seed = to + 1;
46     db = to + SHA_DIGEST_LENGTH + 1;
47 
48     SHA1(param, plen, db);
49     memset(db + SHA_DIGEST_LENGTH, 0,
50 	   emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
51     db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
52     memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, (unsigned int) flen);
53     RAND_bytes(seed, SHA_DIGEST_LENGTH);
54 #ifdef PKCS_TESTVECT
55     memcpy(seed,
56 	   "\xaa\xfd\x12\xf6\x59\xca\xe6\x34\x89\xb4\x79\xe5\x07\x6d\xde\xc2\xf0\x6c\xb5\x8f",
57 	   20);
58 #endif
59 
60     MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
61     for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
62 	db[i] ^= dbmask[i];
63 
64     MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);
65     for (i = 0; i < SHA_DIGEST_LENGTH; i++)
66 	seed[i] ^= seedmask[i];
67 
68     Free(dbmask);
69     return (1);
70     }
71 
72 int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
73 	     unsigned char *from, int flen, int num, unsigned char *param,
74 	     int plen)
75     {
76     int i, dblen, mlen = -1;
77     unsigned char *maskeddb;
78     int lzero;
79     unsigned char *db, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
80 
81     if (--num < 2 * SHA_DIGEST_LENGTH + 1)
82 	{
83 	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
84 	return (-1);
85 	}
86 
87     dblen = num - SHA_DIGEST_LENGTH;
88     db = Malloc(dblen);
89     if (db == NULL)
90 	{
91 	RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
92 	return (-1);
93 	}
94 
95     lzero = num - flen;
96     maskeddb = from - lzero + SHA_DIGEST_LENGTH;
97 
98     MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
99     for (i = lzero; i < SHA_DIGEST_LENGTH; i++)
100 	seed[i] ^= from[i - lzero];
101 
102     MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
103     for (i = 0; i < dblen; i++)
104 	db[i] ^= maskeddb[i];
105 
106     SHA1(param, plen, phash);
107 
108     if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0)
109 	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
110     else
111 	{
112 	for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
113 	    if (db[i] != 0x00)
114 		break;
115 	if (db[i] != 0x01 || i++ >= dblen)
116 	    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
117 		   RSA_R_OAEP_DECODING_ERROR);
118 	else
119 	    {
120 	    mlen = dblen - i;
121 	    if (tlen < mlen)
122 		{
123 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);
124 		mlen = -1;
125 		}
126 	    else
127 		memcpy(to, db + i, mlen);
128 	    }
129 	}
130     Free(db);
131     return (mlen);
132     }
133 
134 int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen)
135     {
136     long i, outlen = 0;
137     unsigned char cnt[4];
138     SHA_CTX c;
139     unsigned char md[SHA_DIGEST_LENGTH];
140 
141     for (i = 0; outlen < len; i++)
142 	{
143 	cnt[0] = (i >> 24) & 255, cnt[1] = (i >> 16) & 255,
144 	  cnt[2] = (i >> 8) & 255, cnt[3] = i & 255;
145 	SHA1_Init(&c);
146 	SHA1_Update(&c, seed, seedlen);
147 	SHA1_Update(&c, cnt, 4);
148 	if (outlen + SHA_DIGEST_LENGTH <= len)
149 	    {
150 	    SHA1_Final(mask + outlen, &c);
151 	    outlen += SHA_DIGEST_LENGTH;
152 	    }
153 	else
154 	    {
155 	    SHA1_Final(md, &c);
156 	    memcpy(mask + outlen, md, len - outlen);
157 	    outlen = len;
158 	    }
159 	}
160     return (0);
161     }
162 #endif
163