xref: /onnv-gate/usr/src/common/net/wanboot/crypt/des3.c (revision 0:68f95e015346)
1*0Sstevel@tonic-gate /*
2*0Sstevel@tonic-gate  * CDDL HEADER START
3*0Sstevel@tonic-gate  *
4*0Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*0Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*0Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*0Sstevel@tonic-gate  * with the License.
8*0Sstevel@tonic-gate  *
9*0Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*0Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*0Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*0Sstevel@tonic-gate  * and limitations under the License.
13*0Sstevel@tonic-gate  *
14*0Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*0Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*0Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*0Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*0Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*0Sstevel@tonic-gate  *
20*0Sstevel@tonic-gate  * CDDL HEADER END
21*0Sstevel@tonic-gate  */
22*0Sstevel@tonic-gate /*
23*0Sstevel@tonic-gate  * Copyright 2002-2003 Sun Microsystems, Inc.  All rights reserved.
24*0Sstevel@tonic-gate  * Use is subject to license terms.
25*0Sstevel@tonic-gate  */
26*0Sstevel@tonic-gate 
27*0Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
28*0Sstevel@tonic-gate 
29*0Sstevel@tonic-gate #include <stdlib.h>
30*0Sstevel@tonic-gate #include <strings.h>
31*0Sstevel@tonic-gate #include <sys/sysmacros.h>
32*0Sstevel@tonic-gate 
33*0Sstevel@tonic-gate #include "des3.h"
34*0Sstevel@tonic-gate #include "des.h"
35*0Sstevel@tonic-gate 
36*0Sstevel@tonic-gate typedef struct keysched_s {
37*0Sstevel@tonic-gate 	uint32_t ksch_encrypt1[16][2];
38*0Sstevel@tonic-gate 	uint32_t ksch_encrypt2[16][2];
39*0Sstevel@tonic-gate 	uint32_t ksch_encrypt3[16][2];
40*0Sstevel@tonic-gate 
41*0Sstevel@tonic-gate 	uint32_t ksch_decrypt1[16][2];
42*0Sstevel@tonic-gate 	uint32_t ksch_decrypt2[16][2];
43*0Sstevel@tonic-gate 	uint32_t ksch_decrypt3[16][2];
44*0Sstevel@tonic-gate } keysched_t;
45*0Sstevel@tonic-gate 
46*0Sstevel@tonic-gate int
47*0Sstevel@tonic-gate des3_init(void **cookie)
48*0Sstevel@tonic-gate {
49*0Sstevel@tonic-gate 	if ((*cookie = malloc(sizeof (keysched_t))) == NULL) {
50*0Sstevel@tonic-gate 		return (-1);
51*0Sstevel@tonic-gate 	}
52*0Sstevel@tonic-gate 	return (0);
53*0Sstevel@tonic-gate }
54*0Sstevel@tonic-gate 
55*0Sstevel@tonic-gate void
56*0Sstevel@tonic-gate des3_fini(void *cookie)
57*0Sstevel@tonic-gate {
58*0Sstevel@tonic-gate 	free(cookie);
59*0Sstevel@tonic-gate }
60*0Sstevel@tonic-gate 
61*0Sstevel@tonic-gate void
62*0Sstevel@tonic-gate des3_encrypt(void *cookie, uint8_t *block)
63*0Sstevel@tonic-gate {
64*0Sstevel@tonic-gate 	keysched_t *ksch = (keysched_t *)cookie;
65*0Sstevel@tonic-gate 
66*0Sstevel@tonic-gate 	des(ksch->ksch_encrypt1, block);
67*0Sstevel@tonic-gate 	des(ksch->ksch_decrypt2, block);
68*0Sstevel@tonic-gate 	des(ksch->ksch_encrypt3, block);
69*0Sstevel@tonic-gate }
70*0Sstevel@tonic-gate 
71*0Sstevel@tonic-gate void
72*0Sstevel@tonic-gate des3_decrypt(void *cookie, uint8_t *block)
73*0Sstevel@tonic-gate {
74*0Sstevel@tonic-gate 	keysched_t *ksch = (keysched_t *)cookie;
75*0Sstevel@tonic-gate 
76*0Sstevel@tonic-gate 	des(ksch->ksch_decrypt3, block);
77*0Sstevel@tonic-gate 	des(ksch->ksch_encrypt2, block);
78*0Sstevel@tonic-gate 	des(ksch->ksch_decrypt1, block);
79*0Sstevel@tonic-gate }
80*0Sstevel@tonic-gate 
81*0Sstevel@tonic-gate /*
82*0Sstevel@tonic-gate  * Generate key schedule for triple DES in E-D-E (or D-E-D) mode.
83*0Sstevel@tonic-gate  *
84*0Sstevel@tonic-gate  * The key argument is taken to be 24 bytes. The first 8 bytes are K1
85*0Sstevel@tonic-gate  * for the first stage, the second 8 bytes are K2 for the middle stage
86*0Sstevel@tonic-gate  * and the third 8 bytes are K3 for the last stage
87*0Sstevel@tonic-gate  */
88*0Sstevel@tonic-gate void
89*0Sstevel@tonic-gate des3_key(void *cookie, const uint8_t *key)
90*0Sstevel@tonic-gate {
91*0Sstevel@tonic-gate 	keysched_t *ks = (keysched_t *)cookie;
92*0Sstevel@tonic-gate 	uint8_t *k1 = (uint8_t *)key;
93*0Sstevel@tonic-gate 	uint8_t *k2 = k1 + DES_KEY_SIZE;
94*0Sstevel@tonic-gate 	uint8_t *k3 = k2 + DES_KEY_SIZE;
95*0Sstevel@tonic-gate 
96*0Sstevel@tonic-gate 	des_key(ks->ksch_decrypt1, k1, B_TRUE);
97*0Sstevel@tonic-gate 	des_key(ks->ksch_encrypt1, k1, B_FALSE);
98*0Sstevel@tonic-gate 	des_key(ks->ksch_decrypt2, k2, B_TRUE);
99*0Sstevel@tonic-gate 	des_key(ks->ksch_encrypt2, k2, B_FALSE);
100*0Sstevel@tonic-gate 	des_key(ks->ksch_decrypt3, k3, B_TRUE);
101*0Sstevel@tonic-gate 	des_key(ks->ksch_encrypt3, k3, B_FALSE);
102*0Sstevel@tonic-gate }
103*0Sstevel@tonic-gate 
104*0Sstevel@tonic-gate 
105*0Sstevel@tonic-gate boolean_t
106*0Sstevel@tonic-gate des3_keycheck(const uint8_t *key)
107*0Sstevel@tonic-gate {
108*0Sstevel@tonic-gate 	uint64_t key_so_far;
109*0Sstevel@tonic-gate 	uint64_t scratch;
110*0Sstevel@tonic-gate 	uint64_t *currentkey;
111*0Sstevel@tonic-gate 	uint64_t tmpbuf[3];
112*0Sstevel@tonic-gate 	uint_t parity;
113*0Sstevel@tonic-gate 	uint_t num_weakkeys = 0;
114*0Sstevel@tonic-gate 	uint_t i;
115*0Sstevel@tonic-gate 	uint_t j;
116*0Sstevel@tonic-gate 
117*0Sstevel@tonic-gate 	/*
118*0Sstevel@tonic-gate 	 * Table of weak and semi-weak keys.  Fortunately, weak keys are
119*0Sstevel@tonic-gate 	 * endian-independent, and some semi-weak keys can be paired up in
120*0Sstevel@tonic-gate 	 * endian-opposite order.  Since keys are stored as uint64_t's,
121*0Sstevel@tonic-gate 	 * use the ifdef _LITTLE_ENDIAN where appropriate.
122*0Sstevel@tonic-gate 	 */
123*0Sstevel@tonic-gate 	static uint64_t des_weak_keys[] = {
124*0Sstevel@tonic-gate 		/* Really weak keys.  Byte-order independent values. */
125*0Sstevel@tonic-gate 		0x0101010101010101,
126*0Sstevel@tonic-gate 		0x1f1f1f1f0e0e0e0e,
127*0Sstevel@tonic-gate 		0xe0e0e0e0f1f1f1f1,
128*0Sstevel@tonic-gate 		0xfefefefefefefefe,
129*0Sstevel@tonic-gate 
130*0Sstevel@tonic-gate 		/* Semi-weak (and a few possibly-weak) keys. */
131*0Sstevel@tonic-gate 
132*0Sstevel@tonic-gate 		/* Byte-order independent semi-weak keys. */
133*0Sstevel@tonic-gate 		0x01fe01fe01fe01fe,	0xfe01fe01fe01fe01,
134*0Sstevel@tonic-gate 
135*0Sstevel@tonic-gate 		/* Byte-order dependent semi-weak keys. */
136*0Sstevel@tonic-gate #ifdef _LITTLE_ENDIAN
137*0Sstevel@tonic-gate 		0xf10ef10ee01fe01f,	0x0ef10ef11fe01fe0,
138*0Sstevel@tonic-gate 		0x01f101f101e001e0,	0xf101f101e001e001,
139*0Sstevel@tonic-gate 		0x0efe0efe1ffe1ffe,	0xfe0efe0efe1ffe1f,
140*0Sstevel@tonic-gate 		0x010e010e011f011f,	0x0e010e011f011f01,
141*0Sstevel@tonic-gate 		0xf1fef1fee0fee0fe,	0xfef1fef1fee0fee0,
142*0Sstevel@tonic-gate #else	/* Big endian */
143*0Sstevel@tonic-gate 		0x1fe01fe00ef10ef1,	0xe01fe01ff10ef10e,
144*0Sstevel@tonic-gate 		0x01e001e001f101f1,	0xe001e001f101f101,
145*0Sstevel@tonic-gate 		0x1ffe1ffe0efe0efe,	0xfe1ffe1ffe0efe0e,
146*0Sstevel@tonic-gate 		0x011f011f010e010e,	0x1f011f010e010e01,
147*0Sstevel@tonic-gate 		0xe0fee0fef1fef1fe,	0xfee0fee0fef1fef1,
148*0Sstevel@tonic-gate #endif
149*0Sstevel@tonic-gate 
150*0Sstevel@tonic-gate 		/* We'll save the other possibly-weak keys for the future. */
151*0Sstevel@tonic-gate 	};
152*0Sstevel@tonic-gate 
153*0Sstevel@tonic-gate 	if (IS_P2ALIGNED(key, sizeof (uint64_t))) {
154*0Sstevel@tonic-gate 		/* LINTED */
155*0Sstevel@tonic-gate 		currentkey = (uint64_t *)key;
156*0Sstevel@tonic-gate 	} else {
157*0Sstevel@tonic-gate 		currentkey = tmpbuf;
158*0Sstevel@tonic-gate 		bcopy(key, currentkey, 3 * sizeof (uint64_t));
159*0Sstevel@tonic-gate 	}
160*0Sstevel@tonic-gate 
161*0Sstevel@tonic-gate 	for (j = 0; j < 3; j++) {
162*0Sstevel@tonic-gate 		key_so_far = currentkey[j];
163*0Sstevel@tonic-gate 		scratch = key_so_far;
164*0Sstevel@tonic-gate 
165*0Sstevel@tonic-gate 		/* Unroll the loop within each byte. */
166*0Sstevel@tonic-gate 		for (i = 0; i < 8; i++) {
167*0Sstevel@tonic-gate 			parity = 1;
168*0Sstevel@tonic-gate 
169*0Sstevel@tonic-gate 			/*
170*0Sstevel@tonic-gate 			 * Start shifting at byte n, right to left.
171*0Sstevel@tonic-gate 			 * Low bit (0) doesn't count.
172*0Sstevel@tonic-gate 			 */
173*0Sstevel@tonic-gate 			scratch >>= 1;
174*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 1 */
175*0Sstevel@tonic-gate 				parity++;
176*0Sstevel@tonic-gate 			scratch >>= 1;
177*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 2 */
178*0Sstevel@tonic-gate 				parity++;
179*0Sstevel@tonic-gate 			scratch >>= 1;
180*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 3 */
181*0Sstevel@tonic-gate 				parity++;
182*0Sstevel@tonic-gate 			scratch >>= 1;
183*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 4 */
184*0Sstevel@tonic-gate 				parity++;
185*0Sstevel@tonic-gate 			scratch >>= 1;
186*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 5 */
187*0Sstevel@tonic-gate 				parity++;
188*0Sstevel@tonic-gate 			scratch >>= 1;
189*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 6 */
190*0Sstevel@tonic-gate 			parity++;
191*0Sstevel@tonic-gate 			scratch >>= 1;
192*0Sstevel@tonic-gate 			if (scratch & 0x1)	/* bit 7 */
193*0Sstevel@tonic-gate 				parity++;
194*0Sstevel@tonic-gate 			scratch >>= 1;
195*0Sstevel@tonic-gate 
196*0Sstevel@tonic-gate 			parity &= 1;	/* Mask off other bits. */
197*0Sstevel@tonic-gate 
198*0Sstevel@tonic-gate 			/* Will common subexpression elimination help me? */
199*0Sstevel@tonic-gate 			key_so_far &= ~((uint64_t)1 << (i << 3));
200*0Sstevel@tonic-gate 			key_so_far |= ((uint64_t)parity << (i << 3));
201*0Sstevel@tonic-gate 		}
202*0Sstevel@tonic-gate 
203*0Sstevel@tonic-gate 		/* Do weak key check itself. */
204*0Sstevel@tonic-gate 		for (i = 0; i < (sizeof (des_weak_keys) / sizeof (uint64_t));
205*0Sstevel@tonic-gate 		    i++) {
206*0Sstevel@tonic-gate 			if (key_so_far == des_weak_keys[i]) {
207*0Sstevel@tonic-gate 				/* In 3DES, one weak key is OK.  Two is bad. */
208*0Sstevel@tonic-gate 				if (++num_weakkeys > 1) {
209*0Sstevel@tonic-gate 					return (B_FALSE);
210*0Sstevel@tonic-gate 				} else {
211*0Sstevel@tonic-gate 					/*
212*0Sstevel@tonic-gate 					 * We found a weak key, but since
213*0Sstevel@tonic-gate 					 * we've only found one weak key,
214*0Sstevel@tonic-gate 					 * we can not reject the whole 3DES
215*0Sstevel@tonic-gate 					 * set of keys as weak.
216*0Sstevel@tonic-gate 					 *
217*0Sstevel@tonic-gate 					 * Break from the weak key loop
218*0Sstevel@tonic-gate 					 * (since this DES key is weak) and
219*0Sstevel@tonic-gate 					 * continue on.
220*0Sstevel@tonic-gate 					 */
221*0Sstevel@tonic-gate 					break;
222*0Sstevel@tonic-gate 				}
223*0Sstevel@tonic-gate 			}
224*0Sstevel@tonic-gate 		}
225*0Sstevel@tonic-gate 
226*0Sstevel@tonic-gate 		/*
227*0Sstevel@tonic-gate 		 * Fix key extension, adjust bits if necessary.
228*0Sstevel@tonic-gate 		 */
229*0Sstevel@tonic-gate 		currentkey[j] = key_so_far;
230*0Sstevel@tonic-gate 	}
231*0Sstevel@tonic-gate 
232*0Sstevel@tonic-gate 	/*
233*0Sstevel@tonic-gate 	 * Perform key equivalence checks, now that parity is properly set.
234*0Sstevel@tonic-gate 	 * All three keys must be unique.
235*0Sstevel@tonic-gate 	 */
236*0Sstevel@tonic-gate 	if (currentkey[0] == currentkey[1] || currentkey[1] == currentkey[2] ||
237*0Sstevel@tonic-gate 	    currentkey[2] == currentkey[0]) {
238*0Sstevel@tonic-gate 		return (B_FALSE);
239*0Sstevel@tonic-gate 	}
240*0Sstevel@tonic-gate 
241*0Sstevel@tonic-gate 	return (B_TRUE);
242*0Sstevel@tonic-gate }
243