xref: /inferno-os/appl/cmd/itest.b (revision 3f1f06c5d12b24c4061e5123acabf72348ff45a2) 
1implement Itest;
2
3include "sys.m";
4	sys: Sys;
5include "string.m";
6	str: String;
7include "draw.m";
8include "daytime.m";
9	daytime: Daytime;
10include "bufio.m";
11	bufio: Bufio;
12	Iobuf: import bufio;
13include "readdir.m";
14	readdir: Readdir;
15include "arg.m";
16include "itslib.m";
17	S_INFO, S_WARN, S_ERROR, S_FATAL, S_STIME, S_ETIME: import Itslib;
18include "env.m";
19	env: Env;
20include "sh.m";
21
22SUMFILE: con "summary";
23MSGFILE: con "msgs";
24README: con "README";
25
26configfile := "";
27cflag := -1;
28verbosity := 3;
29repcount := 1;
30recroot := "";
31display_stderr := 0;
32display_stdout := 0;
33now := 0;
34
35stdout: ref Sys->FD;
36stderr: ref Sys->FD;
37context: ref Draw->Context;
38
39Test: adt {
40	spec: string;
41	fullspec: string;
42	cmd: Command;
43	recdir: string;
44	stdout: string;
45	stderr: string;
46	nruns: int;
47	nwarns: int;
48	nerrors: int;
49	nfatals: int;
50	failed: int;
51};
52
53
54Itest: module
55{
56	init: fn(ctxt: ref Draw->Context, argv: list of string);
57};
58
59
60
61init(ctxt: ref Draw->Context, args: list of string)
62{
63	sys = load Sys Sys->PATH;
64	stdout = sys->fildes(1);
65	stderr = sys->fildes(2);
66	context = ctxt;
67	arg := load Arg Arg->PATH;
68	if(arg == nil)
69		nomod(Arg->PATH);
70	daytime = load Daytime Daytime->PATH;
71	if(daytime == nil)
72		nomod(Daytime->PATH);
73	str = load String String->PATH;
74	bufio = load Bufio Bufio->PATH;
75	if(bufio == nil)
76		nomod(Bufio->PATH);
77	if(str == nil)
78		nomod(String->PATH);
79	readdir = load Readdir Readdir->PATH;
80	if(readdir == nil)
81		nomod(Readdir->PATH);
82	env = load Env Env->PATH;
83	if(env == nil)
84		nomod(Env->PATH);
85	arg->init(args);
86	while((o := arg->opt()) != 0)
87		case o {
88		'c' =>	cflag = toint("c", arg->arg(), 0, 9);
89		'e' =>	display_stderr++;
90		'o' =>	display_stdout++;
91		'r' =>		repcount = toint("r", arg->arg(), 0, -1);
92		'v' =>	verbosity = toint("v", arg->arg(), 0, 9);
93		'C' =>	configfile = arg->arg();
94		'R' =>	recroot = arg->arg();
95		* =>		usage();
96		}
97	args = arg->argv();
98	arg = nil;
99	testlist : array of ref Test;
100	if (args != nil)
101		testlist = arg_tests(args);
102	else if (configfile != "")
103		testlist = config_tests(configfile);
104	if (testlist == nil)
105		fatal("No tests to run");
106	sys->pctl(Sys->FORKENV, nil);
107	if (env->setenv(Itslib->ENV_VERBOSITY, string verbosity))
108		fatal("Failed to set environment variable " + Itslib->ENV_VERBOSITY);
109	if (repcount)
110		reps := string repcount;
111	else
112		reps = "infinite";
113	if (len testlist == 1) ts := "";
114	else ts = "s";
115	if (repcount == 1) rs := "";
116	else rs = "s";
117	mreport(0, S_INFO, 2, sys->sprint("Starting tests - %s run%s of %d test%s", reps, rs, len testlist, ts));
118	run := big 1;
119
120	if (recroot != nil)
121		recn := highest(recroot) + 1;
122	while (repcount == 0 || run <= big repcount) {
123		mreport(1, S_INFO, 3, sys->sprint("Starting run %bd", run));
124		for (i:=0; i<len testlist; i++) {
125			t := testlist[i];
126			if (recroot != nil) {
127				t.recdir = sys->sprint("%s/%d", recroot, recn++);
128				mreport(2, S_INFO, 3, sys->sprint("Recording in %s", t.recdir));
129				rfd := sys->create(t.recdir, Sys->OREAD, Sys->DMDIR | 8r770);
130				if (rfd == nil)
131					fatal(sys->sprint("Failed to create directory %s: %r\n", t.recdir));
132				rfd = nil;
133			}
134			runtest(t);
135		}
136		mreport(1, S_INFO, 3, sys->sprint("Finished run %bd", run));
137		run++;
138	}
139	mreport(0, S_INFO, 2, "Finished tests");
140}
141
142usage()
143{
144	sys->fprint(stderr, "Usage itest [-eo] [-c cflag] [-r count] [-v vlevel] [-C cfile] [-R recroot] [testdir ...]\n");
145	raise "fail: usage";
146}
147
148fatal(s: string)
149{
150	sys->fprint(stderr, "%s\n", s);
151	raise "fail: error";
152}
153
154nomod(mod: string)
155{
156	sys->fprint(stderr, "Failed to load %s\n", mod);
157	raise "fail: module";
158}
159
160toint(opt, s: string, min, max: int): int
161{
162	if (len s == 0 || str->take(s, "[0-9]+-") != s)
163		fatal(sys->sprint("no value specified for option %s", opt));
164	v := int s;
165	if (v < min)
166		fatal(sys->sprint("option %s value is less than minimum of %d: %d", opt, v, min));
167	if (max != -1 && v > max)
168		fatal(sys->sprint("option %s value is greater than maximum of %d: %d", opt, v, max));
169	return v;
170}
171
172arg_tests(args: list of string): array of ref Test
173{
174	al := len args;
175	ta := array[al] of ref Test;
176	for (i:=0; i<al; i++) {
177		tspec := hd args;
178		args = tl args;
179		ta[i] = ref Test(tspec, "", nil, "", "", "", 0, 0, 0, 0, 0);
180		tcheck(ta[i]);
181	}
182	return ta;
183}
184
185config_tests(cf: string): array of ref Test
186{
187	cl := linelist(cf);
188	if (cl == nil)
189		fatal("No tests in config file");
190	al := len cl;
191	ta := array[al] of ref Test;
192	for (i:=0; i<al; i++) {
193		tspec := hd cl;
194		cl = tl cl;
195		ta[i] = ref Test(tspec, "", nil, "", "", "", 0, 0, 0, 0, 0);
196		tcheck(ta[i]);
197	}
198	return ta;
199
200}
201
202highest(path: string): int
203{
204	(da, nd) := readdir->init(path, Readdir->NAME);
205	high := 0;
206	for (i:=0; i<nd; i++) {
207		n := int da[i].name;
208		if (n > high)
209			high = n;
210	}
211	return high;
212}
213
214tcheck(t: ref Test): int
215{
216	td := t.spec;
217	if (!checkdir(td)) {
218		fatal(sys->sprint("Failed to find test %s\n", td));
219		return 0;
220	}
221	tf1 := t.spec + "/t.sh";
222	tf2 := t.spec + "/t.dis";
223	if (checkexec(tf1)) {
224		t.fullspec = tf1;
225		return 1;
226	}
227	if (checkexec(tf2)) {
228		t.fullspec = tf2;
229		return 1;
230	}
231	fatal(sys->sprint("Could not find executable files %s or %s\n", tf1, tf2));
232	return 0;
233}
234
235checkdir(d: string): int
236{
237	(ok, dir) := sys->stat(d);
238	if (ok != 0 || ! dir.qid.qtype & Sys->QTDIR)
239		return 0;
240	return 1;
241}
242
243checkexec(d: string): int
244{
245	(ok, dir) := sys->stat(d);
246	if (ok != 0 || ! dir.mode & 8r100)
247		return 0;
248	return 1;
249}
250
251
252set_cflag(f: int)
253{
254	wfile("/dev/jit", string f, 0);
255
256}
257
258runtest(t: ref Test)
259{
260	if (t.failed)
261		return;
262
263	if (cflag != -1) {
264		mreport(0, S_INFO, 7, sys->sprint("Setting cflag to %d", cflag));
265		set_cflag(cflag);
266	}
267	readme := t.spec + "/" + README;
268	mreport(2, S_INFO, 3, sys->sprint("Starting test %s cflag=%s", t.spec, rfile("/dev/jit")));
269	if (verbosity > 8)
270		display_file(readme);
271	sync := chan of int;
272	spawn monitor(t, sync);
273	<-sync;
274}
275
276monitor(t: ref Test, sync: chan of int)
277{
278	pid := sys->pctl(Sys->FORKFD|Sys->FORKNS|Sys->FORKENV|Sys->NEWPGRP, nil);
279	pa := array[2] of ref Sys->FD;
280	if (sys->pipe(pa))
281		fatal("Failed to set up pipe");
282	if (env->setenv(Itslib->ENV_MFD, string pa[0].fd))
283		fatal("Failed to set environment variable " + Itslib->ENV_MFD);
284	mlfd: ref Sys->FD;
285	if (t.recdir != nil) {
286		mfile := t.recdir+"/"+MSGFILE;
287		mlfd = sys->create(mfile, Sys->OWRITE, 8r660);
288		if (mlfd == nil)
289			fatal(sys->sprint("Failed to create %s: %r'\n", mfile));
290		t.stdout = t.recdir+"/stdout";
291		t.stderr = t.recdir+"/stderr";
292	} else {
293		t.stdout = "/tmp/itest.stdout";
294		t.stderr = "/tmp/itest.stderr";
295	}
296	cf := int rfile("/dev/jit");
297	stime := sys->millisec();
298	swhen := daytime->now();
299	etime := -1;
300	rsync := chan of int;
301	spawn runit(t.fullspec, t.stdout, t.stderr, t.spec, pa[0], rsync);
302	<-rsync;
303	pa[0] = nil;
304	(nwarns, nerrors, nfatals) := (0, 0, 0);
305	while (1) {
306		mbuf := array[Sys->ATOMICIO] of byte;
307		n := sys->read(pa[1], mbuf, len mbuf);
308		if (n <= 0) break;
309		msg := string mbuf[:n];
310		sev := int msg[0:1];
311		verb := int msg[1:2];
312		body := msg[2:];
313		if (sev == S_STIME)
314			stime = int body;
315		else if (sev == S_ETIME)
316			etime = int body;
317		else {
318			if (sev == S_WARN) {
319				nwarns++;
320				t.nwarns++;
321			}
322			else if (sev == S_ERROR) {
323				nerrors++;
324				t.nerrors++;
325			}
326			else if (sev == S_FATAL) {
327				nfatals++;
328				t.nfatals++;
329			}
330			mreport(3, sev, verb, sys->sprint("%s: %s", severs(sev), body));
331		}
332		if (mlfd != nil)
333			sys->fprint(mlfd, "%d:%s", now, msg);
334	}
335	if (etime < 0) {
336		etime = sys->millisec();
337		if (mlfd != nil)
338			sys->fprint(mlfd, "%d:%s", now, sys->sprint("%d0%d\n", S_ETIME, etime));
339	}
340	elapsed := etime-stime;
341	errsum := sys->sprint("WRN:%d ERR:%d FTL:%d", nwarns, nerrors, nfatals);
342	mreport(2, S_INFO, 3, sys->sprint("Finished test %s after %dms - %s", t.spec, elapsed, errsum));
343	if (t.recdir != "") {
344		wfile(t.recdir+"/"+SUMFILE, sys->sprint("%d %d %d %s\n", swhen, elapsed, cf, t.fullspec), 1);
345	}
346	if (display_stdout) {
347		mreport(2, 0, 0, "Stdout from test:");
348		display_file(t.stdout);
349	}
350	if (display_stderr) {
351		mreport(2, 0, 0, "Stderr from test:");
352		display_file(t.stderr);
353	}
354	sync <-= pid;
355}
356
357runit(fullspec, sofile, sefile, tpath: string, mfd: ref Sys->FD, sync: chan of int)
358{
359	pid := sys->pctl(Sys->NEWFD|Sys->FORKNS, mfd.fd::nil);
360	o, e: ref Sys->FD;
361	o = sys->create(sofile, Sys->OWRITE, 8r660);
362	if (o == nil)
363		treport(mfd, S_ERROR, 0, "Failed to open stdout: %r\n");
364	else
365		sys->dup(o.fd, 1);
366	o = nil;
367	e = sys->create(sefile, Sys->OWRITE, 8r660);
368	if (e == nil)
369		treport(mfd, S_ERROR, 0, "Failed to open stderr: %r\n");
370	else
371		sys->dup(e.fd, 2);
372	e = nil;
373	sync <-= pid;
374	args := list of {fullspec};
375	if (fullspec[len fullspec-1] == 's')
376		cmd := load Command fullspec;
377	else {
378		cmd = load Command "/dis/sh.dis";
379		args = fullspec :: args;
380	}
381	if (cmd == nil) {
382		treport(mfd, S_FATAL, 0, sys->sprint("Failed to load Command from %s", "/dis/sh.dis"));
383		return;
384	}
385	if (sys->chdir(tpath))
386		treport(mfd, S_FATAL, 0, "Failed to cd to " + tpath);
387	{
388		cmd->init(context, args);
389	} exception ex {
390		"*" =>
391		treport(mfd, S_FATAL, 0, sys->sprint("Exception %s in test %s", ex, fullspec));
392	}
393}
394
395severs(sevs: int): string
396{
397	SEVMAP :=  array[] of {"INF", "WRN", "ERR", "FTL"};
398	if (sevs >= len SEVMAP)
399		sstr := "UNK";
400	else
401		sstr = SEVMAP[sevs];
402	return sstr;
403}
404
405
406rfile(file: string): string
407{
408	fd := sys->open(file, Sys->OREAD);
409	if (fd == nil) return nil;
410	buf := array[Sys->ATOMICIO] of byte;
411	n := sys->read(fd, buf, len buf);
412	return string buf[:n];
413}
414
415
416wfile(file: string, text: string, create: int): int
417{
418	if (create)
419		fd := sys->create(file, Sys->OWRITE, 8r660);
420	else
421		fd = sys->open(file, Sys->OWRITE);
422	if (fd == nil) {
423		sys->fprint(stderr, "Failed to open %s: %r\n", file);
424		return 0;
425	}
426	a := array of byte text;
427	al := len a;
428	if (sys->write(fd, a, al) != al) {
429		sys->fprint(stderr, "Failed to write to %s: %r\n", file);
430		return 0;
431	}
432	fd = nil;
433	return 1;
434}
435
436linelist(file: string): list of string
437{
438	bf := bufio->open(file, Bufio->OREAD);
439	if (bf == nil)
440		return nil;
441	cl : list of string;
442	while ((line := bf.gets('\n')) != nil) {
443		if (line[len line -1] == '\n')
444			line = line[:len line - 1];
445		cl = line :: cl;
446	}
447	bf = nil;
448	return cl;
449}
450
451display_file(file: string)
452{
453	bf := bufio->open(file, Bufio->OREAD);
454	if (bf == nil)
455		return;
456	while ((line := bf.gets('\n')) != nil) {
457		sys->print("                    %s", line);
458	}
459}
460
461mreport(indent: int, sev: int, verb: int, msg: string)
462{
463	now = daytime->now();
464	tm := daytime->local(now);
465	time := sys->sprint("%4d%02d%02d %02d:%02d:%02d", tm.year+1900, tm.mon-1, tm.mday, tm.hour, tm.min, tm.sec);
466	pad := "---"[:indent];
467	term := "";
468	if (len msg && msg[len msg-1] != '\n')
469		term = "\n";
470	if (sev || verb <= verbosity)
471		sys->print("%s %s%s%s", time, pad, msg, term);
472}
473
474
475treport(mfd: ref Sys->FD, sev: int, verb: int, msg: string)
476{
477	sys->fprint(mfd, "%d%d%s\n", sev, verb, msg);
478}
479