xref: /plan9/sys/src/cmd/vnc/auth.c (revision 9a747e4fd48b9f4522c70c07e8f882a15030f964)
1 #include "vnc.h"
2 #include <libsec.h>
3 #include <auth.h>
4 
5 /*
6  * Encrypt n bytes using the password
7  * as key, padded with zeros to 8 bytes.
8  */
9 enum
10 {
11 	VerLen	= 12
12 };
13 
14 static char version[VerLen+1] = "RFB 003.003\n";
15 
16 static uchar tab[256];
17 
18 /* VNC reverses the bits of each byte before using as a des key */
19 static void
20 mktab(void)
21 {
22 	int i, j, k;
23 	static int once;
24 
25 	if(once)
26 		return;
27 	once = 1;
28 
29 	for(i=0; i<256; i++) {
30 		j=i;
31 		tab[i] = 0;
32 		for(k=0; k<8; k++) {
33 			tab[i] = (tab[i]<<1) | (j&1);
34 			j >>= 1;
35 		}
36 	}
37 }
38 
39 static void
40 vncencrypt(uchar *buf, int n, char *pw)
41 {
42 	uchar *p;
43 	uchar key[9];
44 	DESstate s;
45 
46 	mktab();
47 	memset(key, 0, sizeof key);
48 	strncpy((char*)key, pw, 8);
49 	for(p=key; *p; p++)
50 		*p = tab[*p];
51 
52 	setupDESstate(&s, key, nil);
53 	desECBencrypt(buf, n, &s);
54 }
55 
56 static int
57 readln(char *prompt, char *line, int len)
58 {
59 	char *p;
60 	int fd, ctl, n, nr;
61 
62 	fd = open("/dev/cons", ORDWR);
63 	if(fd < 0)
64 		sysfatal("couldn't open cons");
65 	ctl = open("/dev/consctl", OWRITE);
66 	if(ctl < 0)
67 		sysfatal("couldn't open consctl");
68 	write(ctl, "rawon", 5);
69 	fprint(fd, "%s", prompt);
70 	nr = 0;
71 	p = line;
72 	for(;;){
73 		n = read(fd, p, 1);
74 		if(n < 0){
75 			close(fd);
76 			close(ctl);
77 			return -1;
78 		}
79 		if(n == 0 || *p == '\n' || *p == '\r'){
80 			*p = '\0';
81 			write(fd, "\n", 1);
82 			close(fd);
83 			close(ctl);
84 			return nr;
85 		}
86 		if(*p == '\b'){
87 			if(nr > 0){
88 				nr--;
89 				p--;
90 			}
91 		}else if(*p == 21){		/* cntrl-u */
92 			fprint(fd, "\n%s", prompt);
93 			nr = 0;
94 			p = line;
95 		}else{
96 			nr++;
97 			p++;
98 		}
99 		if(nr == len){
100 			fprint(fd, "line too long; try again\n%s", prompt);
101 			nr = 0;
102 			p = line;
103 		}
104 	}
105 	return -1;
106 }
107 
108 int
109 vnchandshake_srv(Vnc *v)
110 {
111 	char msg[VerLen+1];
112 
113 	strcpy(msg, version);
114 	if(v->extended)
115 		msg[0] = 'E';
116 	if(verbose)
117 		fprint(2, "server version: %s\n", msg);
118 	vncwrbytes(v, msg, VerLen);
119 	vncflush(v);
120 
121 	vncrdbytes(v, msg, VerLen);
122 	if(verbose)
123 		fprint(2, "client version: %s", msg);
124 	return 0;
125 }
126 
127 int
128 vnchandshake(Vnc *v)
129 {
130 	char msg[VerLen+1];
131 
132 	vncrdbytes(v, msg, VerLen);
133 	if(strncmp(msg, "RFB ", 4) == 0)
134 		v->extended = 0;
135 	else if(strncmp(msg, "EFB ", 4) == 0)
136 		v->extended = 1;
137 	else{
138 		werrstr("bad rfb version \"%s\"", msg);
139 		return -1;
140 	}
141 	if(verbose)
142 		fprint(2, "server version %s\n", msg);
143 	strcpy(msg, version);
144 	if(v->extended)
145 		msg[0] = 'E';
146 	vncwrbytes(v, msg, VerLen);
147 	vncflush(v);
148 	return 0;
149 }
150 
151 int
152 vncauth(Vnc *v)
153 {
154 	char pw[128], *reason;
155 	uchar chal[VncChalLen];
156 	ulong auth;
157 
158 	auth = vncrdlong(v);
159 	switch(auth) {
160 	default:
161 		werrstr("unknown auth type 0x%lux", auth);
162 		if(verbose)
163 			fprint(2, "unknown auth type 0x%lux", auth);
164 		return -1;
165 
166 	case AFailed:
167 		reason = vncrdstring(v);
168 		werrstr("%s", reason);
169 		if(verbose)
170 			fprint(2, "auth failed: %s\n", reason);
171 		return -1;
172 
173 	case ANoAuth:
174 		if(verbose)
175 			fprint(2, "no auth needed");
176 		break;
177 
178 	case AVncAuth:
179 		vncrdbytes(v, chal, VncChalLen);
180 
181 		readln("password: ", pw, sizeof(pw));
182 
183 		vncencrypt(chal, VncChalLen, pw);
184 		memset(pw, 0, sizeof pw);
185 		vncwrbytes(v, chal, VncChalLen);
186 		vncflush(v);
187 
188 		auth = vncrdlong(v);
189 		switch(auth) {
190 		default:
191 			werrstr("unknown auth response 0x%lux", auth);
192 			return -1;
193 		case VncAuthFailed:
194 			werrstr("authentication failed");
195 			return -1;
196 		case VncAuthTooMany:
197 			werrstr("authentication failed - too many tries");
198 			return -1;
199 		case VncAuthOK:
200 			break;
201 		}
202 		break;
203 	}
204 	return 0;
205 }
206 
207 int
208 vncauth_srv(Vnc *v)
209 {
210 	Chalstate *c;
211 	struct AuthInfo * ai;
212 
213 	if((c = auth_challenge("proto=vnc role=server user=%q", getuser()))==nil || c->nchal != VncChalLen)
214 		sysfatal("vncchal returned with error\n");
215 	vncwrlong(v, AVncAuth);
216 	vncwrbytes(v, c->chal, VncChalLen);
217 	vncflush(v);
218 
219 	vncrdbytes(v, c->chal, VncChalLen);
220 	c->resp = c->chal;
221 	c->nresp = VncChalLen;
222 	ai = auth_response(c);
223 	auth_freechal(c);
224 	if(ai == nil){
225 		if(verbose)
226 			fprint(2, "VNC auth failed\n");
227 		return -1;
228 	}
229 	auth_freeAI(ai);
230 	vncwrlong(v, VncAuthOK);
231 	vncflush(v);
232 
233 	return 0;
234 }
235 
236