xref: /netbsd-src/lib/libtelnet/enc_des.c (revision 5f7096188587a2c7c95fa3c69b78e1ec9c7923d0)
1 /*-
2  * Copyright (c) 1991 The Regents of the University of California.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #ifndef lint
35 /*static char sccsid[] = "from: @(#)enc_des.c	5.1 (Berkeley) 3/22/91";*/
36 static char rcsid[] = "$Id: enc_des.c,v 1.2 1993/08/01 18:32:44 mycroft Exp $";
37 #endif /* not lint */
38 
39 #if	defined(AUTHENTICATE) && defined(ENCRYPT) && defined(DES_ENCRYPT)
40 #include <arpa/telnet.h>
41 #include <stdio.h>
42 #ifdef	__STDC__
43 #include <stdlib.h>
44 #endif
45 
46 #include "encrypt.h"
47 #include "key-proto.h"
48 #include "misc-proto.h"
49 
50 extern encrypt_debug_mode;
51 
52 #define	CFB	0
53 #define	OFB	1
54 
55 #define	NO_SEND_IV	1
56 #define	NO_RECV_IV	2
57 #define	NO_KEYID	4
58 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
59 #define	SUCCESS		0
60 #define	FAILED		-1
61 
62 
63 struct fb {
64 	Block krbdes_key;
65 	Schedule krbdes_sched;
66 	Block temp_feed;
67 	unsigned char fb_feed[64];
68 	int need_start;
69 	int state[2];
70 	int keyid[2];
71 	int once;
72 	struct stinfo {
73 		Block		str_output;
74 		Block		str_feed;
75 		Block		str_iv;
76 		Block		str_ikey;
77 		Schedule	str_sched;
78 		int		str_index;
79 		int		str_flagshift;
80 	} streams[2];
81 };
82 
83 static struct fb fb[2];
84 
85 struct keyidlist {
86 	char	*keyid;
87 	int	keyidlen;
88 	char	*key;
89 	int	keylen;
90 	int	flags;
91 } keyidlist [] = {
92 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
93 	{ 0, 0, 0, 0, 0 }
94 };
95 
96 #define	KEYFLAG_MASK	03
97 
98 #define	KEYFLAG_NOINIT	00
99 #define	KEYFLAG_INIT	01
100 #define	KEYFLAG_OK	02
101 #define	KEYFLAG_BAD	03
102 
103 #define	KEYFLAG_SHIFT	2
104 
105 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
106 
107 #define	FB64_IV		1
108 #define	FB64_IV_OK	2
109 #define	FB64_IV_BAD	3
110 
111 
112 void fb64_stream_iv P((Block, struct stinfo *));
113 void fb64_init P((struct fb *));
114 int fb64_start P((struct fb *, int, int));
115 int fb64_is P((unsigned char *, int, struct fb *));
116 int fb64_reply P((unsigned char *, int, struct fb *));
117 void fb64_session P((Session_Key *, int, struct fb *));
118 void fb64_stream_key P((Block, struct stinfo *));
119 int fb64_keyid P((int, unsigned char *, int *, struct fb *));
120 
121 	void
122 cfb64_init(server)
123 	int server;
124 {
125 	fb64_init(&fb[CFB]);
126 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
127 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
128 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
129 }
130 
131 	void
132 ofb64_init(server)
133 	int server;
134 {
135 	fb64_init(&fb[OFB]);
136 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
137 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
138 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
139 }
140 
141 	void
142 fb64_init(fbp)
143 	register struct fb *fbp;
144 {
145 	bzero((void *)fbp, sizeof(*fbp));
146 	fbp->state[0] = fbp->state[1] = FAILED;
147 	fbp->fb_feed[0] = IAC;
148 	fbp->fb_feed[1] = SB;
149 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
150 	fbp->fb_feed[3] = ENCRYPT_IS;
151 }
152 
153 /*
154  * Returns:
155  *	-1: some error.  Negotiation is done, encryption not ready.
156  *	 0: Successful, initial negotiation all done.
157  *	 1: successful, negotiation not done yet.
158  *	 2: Not yet.  Other things (like getting the key from
159  *	    Kerberos) have to happen before we can continue.
160  */
161 	int
162 cfb64_start(dir, server)
163 	int dir;
164 	int server;
165 {
166 	return(fb64_start(&fb[CFB], dir, server));
167 }
168 	int
169 ofb64_start(dir, server)
170 	int dir;
171 	int server;
172 {
173 	return(fb64_start(&fb[OFB], dir, server));
174 }
175 
176 	static int
177 fb64_start(fbp, dir, server)
178 	struct fb *fbp;
179 	int dir;
180 	int server;
181 {
182 	Block b;
183 	int x;
184 	unsigned char *p;
185 	register int state;
186 
187 	switch (dir) {
188 	case DIR_DECRYPT:
189 		/*
190 		 * This is simply a request to have the other side
191 		 * start output (our input).  He will negotiate an
192 		 * IV so we need not look for it.
193 		 */
194 		state = fbp->state[dir-1];
195 		if (state == FAILED)
196 			state = IN_PROGRESS;
197 		break;
198 
199 	case DIR_ENCRYPT:
200 		state = fbp->state[dir-1];
201 		if (state == FAILED)
202 			state = IN_PROGRESS;
203 		else if ((state & NO_SEND_IV) == 0)
204 			break;
205 
206 		if (!VALIDKEY(fbp->krbdes_key)) {
207 			fbp->need_start = 1;
208 			break;
209 		}
210 		state &= ~NO_SEND_IV;
211 		state |= NO_RECV_IV;
212 		if (encrypt_debug_mode)
213 			printf("Creating new feed\r\n");
214 		/*
215 		 * Create a random feed and send it over.
216 		 */
217 		des_new_random_key(fbp->temp_feed);
218 		des_ecb_encrypt(fbp->temp_feed, fbp->temp_feed,
219 				fbp->krbdes_sched, 1);
220 		p = fbp->fb_feed + 3;
221 		*p++ = ENCRYPT_IS;
222 		p++;
223 		*p++ = FB64_IV;
224 		for (x = 0; x < sizeof(Block); ++x) {
225 			if ((*p++ = fbp->temp_feed[x]) == IAC)
226 				*p++ = IAC;
227 		}
228 		*p++ = IAC;
229 		*p++ = SE;
230 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
231 		net_write(fbp->fb_feed, p - fbp->fb_feed);
232 		break;
233 	default:
234 		return(FAILED);
235 	}
236 	return(fbp->state[dir-1] = state);
237 }
238 
239 /*
240  * Returns:
241  *	-1: some error.  Negotiation is done, encryption not ready.
242  *	 0: Successful, initial negotiation all done.
243  *	 1: successful, negotiation not done yet.
244  */
245 	int
246 cfb64_is(data, cnt)
247 	unsigned char *data;
248 	int cnt;
249 {
250 	return(fb64_is(data, cnt, &fb[CFB]));
251 }
252 	int
253 ofb64_is(data, cnt)
254 	unsigned char *data;
255 	int cnt;
256 {
257 	return(fb64_is(data, cnt, &fb[OFB]));
258 }
259 
260 	int
261 fb64_is(data, cnt, fbp)
262 	unsigned char *data;
263 	int cnt;
264 	struct fb *fbp;
265 {
266 	int x;
267 	unsigned char *p;
268 	Block b;
269 	register int state = fbp->state[DIR_DECRYPT-1];
270 
271 	if (cnt-- < 1)
272 		goto failure;
273 
274 	switch (*data++) {
275 	case FB64_IV:
276 		if (cnt != sizeof(Block)) {
277 			if (encrypt_debug_mode)
278 				printf("CFB64: initial vector failed on size\r\n");
279 			state = FAILED;
280 			goto failure;
281 		}
282 
283 		if (encrypt_debug_mode)
284 			printf("CFB64: initial vector received\r\n");
285 
286 		if (encrypt_debug_mode)
287 			printf("Initializing Decrypt stream\r\n");
288 
289 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
290 
291 		p = fbp->fb_feed + 3;
292 		*p++ = ENCRYPT_REPLY;
293 		p++;
294 		*p++ = FB64_IV_OK;
295 		*p++ = IAC;
296 		*p++ = SE;
297 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
298 		net_write(fbp->fb_feed, p - fbp->fb_feed);
299 
300 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
301 		break;
302 
303 	default:
304 		if (encrypt_debug_mode) {
305 			printf("Unknown option type: %d\r\n", *(data-1));
306 			printd(data, cnt);
307 			printf("\r\n");
308 		}
309 		/* FALL THROUGH */
310 	failure:
311 		/*
312 		 * We failed.  Send an FB64_IV_BAD option
313 		 * to the other side so it will know that
314 		 * things failed.
315 		 */
316 		p = fbp->fb_feed + 3;
317 		*p++ = ENCRYPT_REPLY;
318 		p++;
319 		*p++ = FB64_IV_BAD;
320 		*p++ = IAC;
321 		*p++ = SE;
322 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
323 		net_write(fbp->fb_feed, p - fbp->fb_feed);
324 
325 		break;
326 	}
327 	return(fbp->state[DIR_DECRYPT-1] = state);
328 }
329 
330 /*
331  * Returns:
332  *	-1: some error.  Negotiation is done, encryption not ready.
333  *	 0: Successful, initial negotiation all done.
334  *	 1: successful, negotiation not done yet.
335  */
336 	int
337 cfb64_reply(data, cnt)
338 	unsigned char *data;
339 	int cnt;
340 {
341 	return(fb64_reply(data, cnt, &fb[CFB]));
342 }
343 	int
344 ofb64_reply(data, cnt)
345 	unsigned char *data;
346 	int cnt;
347 {
348 	return(fb64_reply(data, cnt, &fb[OFB]));
349 }
350 
351 
352 	int
353 fb64_reply(data, cnt, fbp)
354 	unsigned char *data;
355 	int cnt;
356 	struct fb *fbp;
357 {
358 	int x;
359 	unsigned char *p;
360 	Block b;
361 	register int state = fbp->state[DIR_ENCRYPT-1];
362 
363 	if (cnt-- < 1)
364 		goto failure;
365 
366 	switch (*data++) {
367 	case FB64_IV_OK:
368 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
369 		if (state == FAILED)
370 			state = IN_PROGRESS;
371 		state &= ~NO_RECV_IV;
372 		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
373 		break;
374 
375 	case FB64_IV_BAD:
376 		bzero(fbp->temp_feed, sizeof(Block));
377 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
378 		state = FAILED;
379 		break;
380 
381 	default:
382 		if (encrypt_debug_mode) {
383 			printf("Unknown option type: %d\r\n", data[-1]);
384 			printd(data, cnt);
385 			printf("\r\n");
386 		}
387 		/* FALL THROUGH */
388 	failure:
389 		state = FAILED;
390 		break;
391 	}
392 	return(fbp->state[DIR_ENCRYPT-1] = state);
393 }
394 
395 	void
396 cfb64_session(key, server)
397 	Session_Key *key;
398 	int server;
399 {
400 	fb64_session(key, server, &fb[CFB]);
401 }
402 
403 	void
404 ofb64_session(key, server)
405 	Session_Key *key;
406 	int server;
407 {
408 	fb64_session(key, server, &fb[OFB]);
409 }
410 
411 	static void
412 fb64_session(key, server, fbp)
413 	Session_Key *key;
414 	int server;
415 	struct fb *fbp;
416 {
417 
418 	if (!key || key->type != SK_DES) {
419 		if (encrypt_debug_mode)
420 			printf("Can't set krbdes's session key (%d != %d)\r\n",
421 				key ? key->type : -1, SK_DES);
422 		return;
423 	}
424 	bcopy((void *)key->data, (void *)fbp->krbdes_key, sizeof(Block));
425 
426 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
427 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
428 
429 	if (fbp->once == 0) {
430 		des_set_random_generator_seed(fbp->krbdes_key);
431 		fbp->once = 1;
432 	}
433 	des_key_sched(fbp->krbdes_key, fbp->krbdes_sched);
434 	/*
435 	 * Now look to see if krbdes_start() was was waiting for
436 	 * the key to show up.  If so, go ahead an call it now
437 	 * that we have the key.
438 	 */
439 	if (fbp->need_start) {
440 		fbp->need_start = 0;
441 		fb64_start(fbp, DIR_ENCRYPT, server);
442 	}
443 }
444 
445 /*
446  * We only accept a keyid of 0.  If we get a keyid of
447  * 0, then mark the state as SUCCESS.
448  */
449 	int
450 cfb64_keyid(dir, kp, lenp)
451 	int dir, *lenp;
452 	unsigned char *kp;
453 {
454 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
455 }
456 
457 	int
458 ofb64_keyid(dir, kp, lenp)
459 	int dir, *lenp;
460 	unsigned char *kp;
461 {
462 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
463 }
464 
465 	int
466 fb64_keyid(dir, kp, lenp, fbp)
467 	int dir, *lenp;
468 	unsigned char *kp;
469 	struct fb *fbp;
470 {
471 	register int state = fbp->state[dir-1];
472 
473 	if (*lenp != 1 || (*kp != '\0')) {
474 		*lenp = 0;
475 		return(state);
476 	}
477 
478 	if (state == FAILED)
479 		state = IN_PROGRESS;
480 
481 	state &= ~NO_KEYID;
482 
483 	return(fbp->state[dir-1] = state);
484 }
485 
486 	void
487 fb64_printsub(data, cnt, buf, buflen, type)
488 	unsigned char *data, *buf, *type;
489 	int cnt, buflen;
490 {
491 	char lbuf[32];
492 	register int i;
493 	char *cp;
494 
495 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
496 	buflen -= 1;
497 
498 	switch(data[2]) {
499 	case FB64_IV:
500 		sprintf(lbuf, "%s_IV", type);
501 		cp = lbuf;
502 		goto common;
503 
504 	case FB64_IV_OK:
505 		sprintf(lbuf, "%s_IV_OK", type);
506 		cp = lbuf;
507 		goto common;
508 
509 	case FB64_IV_BAD:
510 		sprintf(lbuf, "%s_IV_BAD", type);
511 		cp = lbuf;
512 		goto common;
513 
514 	default:
515 		sprintf(lbuf, " %d (unknown)", data[2]);
516 		cp = lbuf;
517 	common:
518 		for (; (buflen > 0) && (*buf = *cp++); buf++)
519 			buflen--;
520 		for (i = 3; i < cnt; i++) {
521 			sprintf(lbuf, " %d", data[i]);
522 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
523 				buflen--;
524 		}
525 		break;
526 	}
527 }
528 
529 	void
530 cfb64_printsub(data, cnt, buf, buflen)
531 	unsigned char *data, *buf;
532 	int cnt, buflen;
533 {
534 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
535 }
536 
537 	void
538 ofb64_printsub(data, cnt, buf, buflen)
539 	unsigned char *data, *buf;
540 	int cnt, buflen;
541 {
542 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
543 }
544 
545 	void
546 fb64_stream_iv(seed, stp)
547 	Block seed;
548 	register struct stinfo *stp;
549 {
550 
551 	bcopy((void *)seed, (void *)stp->str_iv, sizeof(Block));
552 	bcopy((void *)seed, (void *)stp->str_output, sizeof(Block));
553 
554 	des_key_sched(stp->str_ikey, stp->str_sched);
555 
556 	stp->str_index = sizeof(Block);
557 }
558 
559 	void
560 fb64_stream_key(key, stp)
561 	Block key;
562 	register struct stinfo *stp;
563 {
564 	bcopy((void *)key, (void *)stp->str_ikey, sizeof(Block));
565 	des_key_sched(key, stp->str_sched);
566 
567 	bcopy((void *)stp->str_iv, (void *)stp->str_output, sizeof(Block));
568 
569 	stp->str_index = sizeof(Block);
570 }
571 
572 /*
573  * DES 64 bit Cipher Feedback
574  *
575  *     key --->+-----+
576  *          +->| DES |--+
577  *          |  +-----+  |
578  *	    |           v
579  *  INPUT --(--------->(+)+---> DATA
580  *          |             |
581  *	    +-------------+
582  *
583  *
584  * Given:
585  *	iV: Initial vector, 64 bits (8 bytes) long.
586  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
587  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
588  *
589  *	V0 = DES(iV, key)
590  *	On = Dn ^ Vn
591  *	V(n+1) = DES(On, key)
592  */
593 
594 	void
595 cfb64_encrypt(s, c)
596 	register unsigned char *s;
597 	int c;
598 {
599 	register struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
600 	register int index;
601 
602 	index = stp->str_index;
603 	while (c-- > 0) {
604 		if (index == sizeof(Block)) {
605 			Block b;
606 			des_ecb_encrypt(stp->str_output, b, stp->str_sched, 1);
607 			bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
608 			index = 0;
609 		}
610 
611 		/* On encryption, we store (feed ^ data) which is cypher */
612 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
613 		s++;
614 		index++;
615 	}
616 	stp->str_index = index;
617 }
618 
619 	int
620 cfb64_decrypt(data)
621 	int data;
622 {
623 	register struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
624 	int index;
625 
626 	if (data == -1) {
627 		/*
628 		 * Back up one byte.  It is assumed that we will
629 		 * never back up more than one byte.  If we do, this
630 		 * may or may not work.
631 		 */
632 		if (stp->str_index)
633 			--stp->str_index;
634 		return(0);
635 	}
636 
637 	index = stp->str_index++;
638 	if (index == sizeof(Block)) {
639 		Block b;
640 		des_ecb_encrypt(stp->str_output, b, stp->str_sched, 1);
641 		bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
642 		stp->str_index = 1;	/* Next time will be 1 */
643 		index = 0;		/* But now use 0 */
644 	}
645 
646 	/* On decryption we store (data) which is cypher. */
647 	stp->str_output[index] = data;
648 	return(data ^ stp->str_feed[index]);
649 }
650 
651 /*
652  * DES 64 bit Output Feedback
653  *
654  * key --->+-----+
655  *	+->| DES |--+
656  *	|  +-----+  |
657  *	+-----------+
658  *	            v
659  *  INPUT -------->(+) ----> DATA
660  *
661  * Given:
662  *	iV: Initial vector, 64 bits (8 bytes) long.
663  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
664  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
665  *
666  *	V0 = DES(iV, key)
667  *	V(n+1) = DES(Vn, key)
668  *	On = Dn ^ Vn
669  */
670 	void
671 ofb64_encrypt(s, c)
672 	register unsigned char *s;
673 	int c;
674 {
675 	register struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
676 	register int index;
677 
678 	index = stp->str_index;
679 	while (c-- > 0) {
680 		if (index == sizeof(Block)) {
681 			Block b;
682 			des_ecb_encrypt(stp->str_feed, b, stp->str_sched, 1);
683 			bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
684 			index = 0;
685 		}
686 		*s++ ^= stp->str_feed[index];
687 		index++;
688 	}
689 	stp->str_index = index;
690 }
691 
692 	int
693 ofb64_decrypt(data)
694 	int data;
695 {
696 	register struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
697 	int index;
698 
699 	if (data == -1) {
700 		/*
701 		 * Back up one byte.  It is assumed that we will
702 		 * never back up more than one byte.  If we do, this
703 		 * may or may not work.
704 		 */
705 		if (stp->str_index)
706 			--stp->str_index;
707 		return(0);
708 	}
709 
710 	index = stp->str_index++;
711 	if (index == sizeof(Block)) {
712 		Block b;
713 		des_ecb_encrypt(stp->str_feed, b, stp->str_sched, 1);
714 		bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
715 		stp->str_index = 1;	/* Next time will be 1 */
716 		index = 0;		/* But now use 0 */
717 	}
718 
719 	return(data ^ stp->str_feed[index]);
720 }
721 #endif
722