1 /* $NetBSD: tinytest.c,v 1.1.1.1 2013/04/11 16:43:32 christos Exp $ */
2 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * 3. The name of the author may not be used to endorse or promote products
13 * derived from this software without specific prior written permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <assert.h>
31
32 #ifdef TINYTEST_LOCAL
33 #include "tinytest_local.h"
34 #endif
35
36 #ifdef WIN32
37 #include <windows.h>
38 #else
39 #include <sys/types.h>
40 #include <sys/wait.h>
41 #include <unistd.h>
42 #endif
43
44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47 /* Workaround for a stupid bug in OSX 10.6 */
48 #define FORK_BREAKS_GCOV
49 #include <vproc.h>
50 #endif
51 #endif
52
53 #ifndef __GNUC__
54 #define __attribute__(x)
55 #endif
56
57 #include "tinytest.h"
58 #include "tinytest_macros.h"
59
60 #define LONGEST_TEST_NAME 16384
61
62 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
63 static int n_ok = 0; /**< Number of tests that have passed */
64 static int n_bad = 0; /**< Number of tests that have failed. */
65 static int n_skipped = 0; /**< Number of tests that have been skipped. */
66
67 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
68 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
69 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
70 const char *verbosity_flag = "";
71
72 enum outcome { SKIP=2, OK=1, FAIL=0 };
73 static enum outcome cur_test_outcome = 0;
74 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
75 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
76 const char *cur_test_name = NULL;
77
78 #ifdef WIN32
79 /* Copy of argv[0] for win32. */
80 static char commandname[MAX_PATH+1];
81 #endif
82
83 static void usage(struct testgroup_t *groups, int list_groups)
84 __attribute__((noreturn));
85
86 static enum outcome
_testcase_run_bare(const struct testcase_t * testcase)87 _testcase_run_bare(const struct testcase_t *testcase)
88 {
89 void *env = NULL;
90 int outcome;
91 if (testcase->setup) {
92 env = testcase->setup->setup_fn(testcase);
93 if (!env)
94 return FAIL;
95 else if (env == (void*)TT_SKIP)
96 return SKIP;
97 }
98
99 cur_test_outcome = OK;
100 testcase->fn(env);
101 outcome = cur_test_outcome;
102
103 if (testcase->setup) {
104 if (testcase->setup->cleanup_fn(testcase, env) == 0)
105 outcome = FAIL;
106 }
107
108 return outcome;
109 }
110
111 #define MAGIC_EXITCODE 42
112
113 static enum outcome
_testcase_run_forked(const struct testgroup_t * group,const struct testcase_t * testcase)114 _testcase_run_forked(const struct testgroup_t *group,
115 const struct testcase_t *testcase)
116 {
117 #ifdef WIN32
118 /* Fork? On Win32? How primitive! We'll do what the smart kids do:
119 we'll invoke our own exe (whose name we recall from the command
120 line) with a command line that tells it to run just the test we
121 want, and this time without forking.
122
123 (No, threads aren't an option. The whole point of forking is to
124 share no state between tests.)
125 */
126 int ok;
127 char buffer[LONGEST_TEST_NAME+256];
128 STARTUPINFOA si;
129 PROCESS_INFORMATION info;
130 DWORD exitcode;
131
132 if (!in_tinytest_main) {
133 printf("\nERROR. On Windows, _testcase_run_forked must be"
134 " called from within tinytest_main.\n");
135 abort();
136 }
137 if (opt_verbosity>0)
138 printf("[forking] ");
139
140 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
141 commandname, verbosity_flag, group->prefix, testcase->name);
142
143 memset(&si, 0, sizeof(si));
144 memset(&info, 0, sizeof(info));
145 si.cb = sizeof(si);
146
147 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
148 0, NULL, NULL, &si, &info);
149 if (!ok) {
150 printf("CreateProcess failed!\n");
151 return 0;
152 }
153 WaitForSingleObject(info.hProcess, INFINITE);
154 GetExitCodeProcess(info.hProcess, &exitcode);
155 CloseHandle(info.hProcess);
156 CloseHandle(info.hThread);
157 if (exitcode == 0)
158 return OK;
159 else if (exitcode == MAGIC_EXITCODE)
160 return SKIP;
161 else
162 return FAIL;
163 #else
164 int outcome_pipe[2];
165 pid_t pid;
166 (void)group;
167
168 if (pipe(outcome_pipe))
169 perror("opening pipe");
170
171 if (opt_verbosity>0)
172 printf("[forking] ");
173 pid = fork();
174 #ifdef FORK_BREAKS_GCOV
175 vproc_transaction_begin(0);
176 #endif
177 if (!pid) {
178 /* child. */
179 int test_r, write_r;
180 char b[1];
181 close(outcome_pipe[0]);
182 test_r = _testcase_run_bare(testcase);
183 assert(0<=(int)test_r && (int)test_r<=2);
184 b[0] = "NYS"[test_r];
185 write_r = (int)write(outcome_pipe[1], b, 1);
186 if (write_r != 1) {
187 perror("write outcome to pipe");
188 exit(1);
189 }
190 exit(0);
191 return FAIL; /* unreachable */
192 } else {
193 /* parent */
194 int status, r;
195 char b[1];
196 /* Close this now, so that if the other side closes it,
197 * our read fails. */
198 close(outcome_pipe[1]);
199 r = (int)read(outcome_pipe[0], b, 1);
200 if (r == 0) {
201 printf("[Lost connection!] ");
202 return 0;
203 } else if (r != 1) {
204 perror("read outcome from pipe");
205 }
206 waitpid(pid, &status, 0);
207 close(outcome_pipe[0]);
208 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
209 }
210 #endif
211 }
212
213 int
testcase_run_one(const struct testgroup_t * group,const struct testcase_t * testcase)214 testcase_run_one(const struct testgroup_t *group,
215 const struct testcase_t *testcase)
216 {
217 enum outcome outcome;
218
219 if (testcase->flags & TT_SKIP) {
220 if (opt_verbosity>0)
221 printf("%s%s: SKIPPED\n",
222 group->prefix, testcase->name);
223 ++n_skipped;
224 return SKIP;
225 }
226
227 if (opt_verbosity>0 && !opt_forked) {
228 printf("%s%s: ", group->prefix, testcase->name);
229 } else {
230 if (opt_verbosity==0) printf(".");
231 cur_test_prefix = group->prefix;
232 cur_test_name = testcase->name;
233 }
234
235 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
236 outcome = _testcase_run_forked(group, testcase);
237 } else {
238 outcome = _testcase_run_bare(testcase);
239 }
240
241 if (outcome == OK) {
242 ++n_ok;
243 if (opt_verbosity>0 && !opt_forked)
244 puts(opt_verbosity==1?"OK":"");
245 } else if (outcome == SKIP) {
246 ++n_skipped;
247 if (opt_verbosity>0 && !opt_forked)
248 puts("SKIPPED");
249 } else {
250 ++n_bad;
251 if (!opt_forked)
252 printf("\n [%s FAILED]\n", testcase->name);
253 }
254
255 if (opt_forked) {
256 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
257 return 1; /* unreachable */
258 } else {
259 return (int)outcome;
260 }
261 }
262
263 int
_tinytest_set_flag(struct testgroup_t * groups,const char * arg,unsigned long flag)264 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag)
265 {
266 int i, j;
267 size_t length = LONGEST_TEST_NAME;
268 char fullname[LONGEST_TEST_NAME];
269 int found=0;
270 if (strstr(arg, ".."))
271 length = strstr(arg,"..")-arg;
272 for (i=0; groups[i].prefix; ++i) {
273 for (j=0; groups[i].cases[j].name; ++j) {
274 snprintf(fullname, sizeof(fullname), "%s%s",
275 groups[i].prefix, groups[i].cases[j].name);
276 if (!flag) /* Hack! */
277 printf(" %s\n", fullname);
278 if (!strncmp(fullname, arg, length)) {
279 groups[i].cases[j].flags |= flag;
280 ++found;
281 }
282 }
283 }
284 return found;
285 }
286
287 static void
usage(struct testgroup_t * groups,int list_groups)288 usage(struct testgroup_t *groups, int list_groups)
289 {
290 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
291 puts(" Specify tests by name, or using a prefix ending with '..'");
292 puts(" To skip a test, list give its name prefixed with a colon.");
293 puts(" Use --list-tests for a list of tests.");
294 if (list_groups) {
295 puts("Known tests are:");
296 _tinytest_set_flag(groups, "..", 0);
297 }
298 exit(0);
299 }
300
301 int
tinytest_main(int c,const char ** v,struct testgroup_t * groups)302 tinytest_main(int c, const char **v, struct testgroup_t *groups)
303 {
304 int i, j, n=0;
305
306 #ifdef WIN32
307 const char *sp = strrchr(v[0], '.');
308 const char *extension = "";
309 if (!sp || stricmp(sp, ".exe"))
310 extension = ".exe"; /* Add an exe so CreateProcess will work */
311 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
312 commandname[MAX_PATH]='\0';
313 #endif
314 for (i=1; i<c; ++i) {
315 if (v[i][0] == '-') {
316 if (!strcmp(v[i], "--RUNNING-FORKED")) {
317 opt_forked = 1;
318 } else if (!strcmp(v[i], "--no-fork")) {
319 opt_nofork = 1;
320 } else if (!strcmp(v[i], "--quiet")) {
321 opt_verbosity = -1;
322 verbosity_flag = "--quiet";
323 } else if (!strcmp(v[i], "--verbose")) {
324 opt_verbosity = 2;
325 verbosity_flag = "--verbose";
326 } else if (!strcmp(v[i], "--terse")) {
327 opt_verbosity = 0;
328 verbosity_flag = "--terse";
329 } else if (!strcmp(v[i], "--help")) {
330 usage(groups, 0);
331 } else if (!strcmp(v[i], "--list-tests")) {
332 usage(groups, 1);
333 } else {
334 printf("Unknown option %s. Try --help\n",v[i]);
335 return -1;
336 }
337 } else {
338 const char *test = v[i];
339 int flag = _TT_ENABLED;
340 if (test[0] == ':') {
341 ++test;
342 flag = TT_SKIP;
343 } else {
344 ++n;
345 }
346 if (!_tinytest_set_flag(groups, test, flag)) {
347 printf("No such test as %s!\n", v[i]);
348 return -1;
349 }
350 }
351 }
352 if (!n)
353 _tinytest_set_flag(groups, "..", _TT_ENABLED);
354
355 setvbuf(stdout, NULL, _IONBF, 0);
356
357 ++in_tinytest_main;
358 for (i=0; groups[i].prefix; ++i)
359 for (j=0; groups[i].cases[j].name; ++j)
360 if (groups[i].cases[j].flags & _TT_ENABLED)
361 testcase_run_one(&groups[i],
362 &groups[i].cases[j]);
363
364 --in_tinytest_main;
365
366 if (opt_verbosity==0)
367 puts("");
368
369 if (n_bad)
370 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
371 n_bad+n_ok,n_skipped);
372 else if (opt_verbosity >= 1)
373 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
374
375 return (n_bad == 0) ? 0 : 1;
376 }
377
378 int
_tinytest_get_verbosity(void)379 _tinytest_get_verbosity(void)
380 {
381 return opt_verbosity;
382 }
383
384 void
_tinytest_set_test_failed(void)385 _tinytest_set_test_failed(void)
386 {
387 if (opt_verbosity <= 0 && cur_test_name) {
388 if (opt_verbosity==0) puts("");
389 printf("%s%s: ", cur_test_prefix, cur_test_name);
390 cur_test_name = NULL;
391 }
392 cur_test_outcome = 0;
393 }
394
395 void
_tinytest_set_test_skipped(void)396 _tinytest_set_test_skipped(void)
397 {
398 if (cur_test_outcome==OK)
399 cur_test_outcome = SKIP;
400 }
401
402