1 /* $NetBSD: tinytest.c,v 1.1.1.3 2021/04/07 02:43:15 christos Exp $ */
2 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * 3. The name of the author may not be used to endorse or promote products
13 * derived from this software without specific prior written permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26 #ifdef TINYTEST_LOCAL
27 #include "tinytest_local.h"
28 #endif
29
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <assert.h>
34
35 #ifndef NO_FORKING
36
37 #ifdef _WIN32
38 #include <windows.h>
39 #else
40 #include <sys/types.h>
41 #include <sys/wait.h>
42 #include <unistd.h>
43 #endif
44
45 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
46 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
47 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
48 /* Workaround for a stupid bug in OSX 10.6 */
49 #define FORK_BREAKS_GCOV
50 #include <vproc.h>
51 #endif
52 #endif
53
54 #endif /* !NO_FORKING */
55
56 #ifndef __GNUC__
57 #define __attribute__(x)
58 #endif
59
60 #include "tinytest.h"
61 #include "tinytest_macros.h"
62
63 #define LONGEST_TEST_NAME 16384
64 #define DEFAULT_TESTCASE_TIMEOUT 30U
65 #define MAGIC_EXITCODE 42
66
67 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
68 static int n_ok = 0; /**< Number of tests that have passed */
69 static int n_bad = 0; /**< Number of tests that have failed. */
70 static int n_skipped = 0; /**< Number of tests that have been skipped. */
71
72 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
73 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
74 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
75 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
76 const char *verbosity_flag = "";
77
78 const struct testlist_alias_t *cfg_aliases=NULL;
79
80 enum outcome { SKIP=2, OK=1, FAIL=0 };
81 static enum outcome cur_test_outcome = 0;
82 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
83 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
84 const char *cur_test_name = NULL;
85
86 static void usage(struct testgroup_t *groups, int list_groups)
87 __attribute__((noreturn));
88 static int process_test_option(struct testgroup_t *groups, const char *test);
89
90 #ifdef _WIN32
91 /* Copy of argv[0] for win32. */
92 static char commandname[MAX_PATH+1];
93
94 struct timeout_thread_args {
95 const testcase_fn *fn;
96 void *env;
97 };
98
99 static DWORD WINAPI
timeout_thread_proc_(LPVOID arg)100 timeout_thread_proc_(LPVOID arg)
101 {
102 struct timeout_thread_args *args = arg;
103 (*(args->fn))(args->env);
104 ExitThread(cur_test_outcome == FAIL ? 1 : 0);
105 }
106
107 static enum outcome
testcase_run_in_thread_(const struct testcase_t * testcase,void * env)108 testcase_run_in_thread_(const struct testcase_t *testcase, void *env)
109 {
110 /* We will never run testcase in a new thread when the
111 timeout is set to zero */
112 assert(opt_timeout);
113 DWORD ret, tid;
114 HANDLE handle;
115 struct timeout_thread_args args = {
116 &(testcase->fn),
117 env
118 };
119
120 handle =CreateThread(NULL, 0, timeout_thread_proc_,
121 (LPVOID)&args, 0, &tid);
122 ret = WaitForSingleObject(handle, opt_timeout * 1000U);
123 if (ret == WAIT_OBJECT_0) {
124 ret = 0;
125 if (!GetExitCodeThread(handle, &ret)) {
126 printf("GetExitCodeThread failed\n");
127 ret = 1;
128 }
129 } else if (ret == WAIT_TIMEOUT) {
130 printf("timeout\n");
131 } else {
132 printf("Wait failed\n");
133 }
134 CloseHandle(handle);
135 if (ret == 0)
136 return OK;
137 else if (ret == MAGIC_EXITCODE)
138 return SKIP;
139 else
140 return FAIL;
141 }
142 #else
testcase_set_timeout_(void)143 static unsigned int testcase_set_timeout_(void)
144 {
145 return alarm(opt_timeout);
146 }
147
testcase_reset_timeout_(void)148 static unsigned int testcase_reset_timeout_(void)
149 {
150 return alarm(0);
151 }
152 #endif
153
154 static enum outcome
testcase_run_bare_(const struct testcase_t * testcase)155 testcase_run_bare_(const struct testcase_t *testcase)
156 {
157 void *env = NULL;
158 int outcome;
159 if (testcase->setup) {
160 env = testcase->setup->setup_fn(testcase);
161 if (!env)
162 return FAIL;
163 else if (env == (void*)TT_SKIP)
164 return SKIP;
165 }
166
167 cur_test_outcome = OK;
168 {
169 if (opt_timeout) {
170 #ifdef _WIN32
171 cur_test_outcome = testcase_run_in_thread_(testcase, env);
172 #else
173 testcase_set_timeout_();
174 testcase->fn(env);
175 testcase_reset_timeout_();
176 #endif
177 } else {
178 testcase->fn(env);
179 }
180 }
181 outcome = cur_test_outcome;
182
183 if (testcase->setup) {
184 if (testcase->setup->cleanup_fn(testcase, env) == 0)
185 outcome = FAIL;
186 }
187
188 return outcome;
189 }
190
191
192 #ifndef NO_FORKING
193
194 static enum outcome
testcase_run_forked_(const struct testgroup_t * group,const struct testcase_t * testcase)195 testcase_run_forked_(const struct testgroup_t *group,
196 const struct testcase_t *testcase)
197 {
198 #ifdef _WIN32
199 /* Fork? On Win32? How primitive! We'll do what the smart kids do:
200 we'll invoke our own exe (whose name we recall from the command
201 line) with a command line that tells it to run just the test we
202 want, and this time without forking.
203
204 (No, threads aren't an option. The whole point of forking is to
205 share no state between tests.)
206 */
207 int ok;
208 char buffer[LONGEST_TEST_NAME+256];
209 STARTUPINFOA si;
210 PROCESS_INFORMATION info;
211 DWORD ret;
212
213 if (!in_tinytest_main) {
214 printf("\nERROR. On Windows, testcase_run_forked_ must be"
215 " called from within tinytest_main.\n");
216 abort();
217 }
218 if (opt_verbosity>0)
219 printf("[forking] ");
220
221 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s --timeout 0 %s%s",
222 commandname, verbosity_flag, group->prefix, testcase->name);
223
224 memset(&si, 0, sizeof(si));
225 memset(&info, 0, sizeof(info));
226 si.cb = sizeof(si);
227
228 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
229 0, NULL, NULL, &si, &info);
230 if (!ok) {
231 printf("CreateProcess failed!\n");
232 return FAIL;
233 }
234 ret = WaitForSingleObject(info.hProcess,
235 (opt_timeout ? opt_timeout * 1000U : INFINITE));
236
237 if (ret == WAIT_OBJECT_0) {
238 GetExitCodeProcess(info.hProcess, &ret);
239 } else if (ret == WAIT_TIMEOUT) {
240 printf("timeout\n");
241 } else {
242 printf("Wait failed\n");
243 }
244 CloseHandle(info.hProcess);
245 CloseHandle(info.hThread);
246 if (ret == 0)
247 return OK;
248 else if (ret == MAGIC_EXITCODE)
249 return SKIP;
250 else
251 return FAIL;
252 #else
253 int outcome_pipe[2];
254 pid_t pid;
255 (void)group;
256
257 if (pipe(outcome_pipe))
258 perror("opening pipe");
259
260 if (opt_verbosity>0)
261 printf("[forking] ");
262 pid = fork();
263 #ifdef FORK_BREAKS_GCOV
264 vproc_transaction_begin(0);
265 #endif
266 if (!pid) {
267 /* child. */
268 int test_r, write_r;
269 char b[1];
270 close(outcome_pipe[0]);
271 test_r = testcase_run_bare_(testcase);
272 assert(0<=(int)test_r && (int)test_r<=2);
273 b[0] = "NYS"[test_r];
274 write_r = (int)write(outcome_pipe[1], b, 1);
275 if (write_r != 1) {
276 perror("write outcome to pipe");
277 exit(1);
278 }
279 exit(0);
280 return FAIL; /* unreachable */
281 } else {
282 /* parent */
283 int status, r, exitcode;
284 char b[1];
285 /* Close this now, so that if the other side closes it,
286 * our read fails. */
287 close(outcome_pipe[1]);
288 r = (int)read(outcome_pipe[0], b, 1);
289 if (r == 0) {
290 printf("[Lost connection!] ");
291 return FAIL;
292 } else if (r != 1) {
293 perror("read outcome from pipe");
294 }
295 waitpid(pid, &status, 0);
296 exitcode = WEXITSTATUS(status);
297 close(outcome_pipe[0]);
298 if (opt_verbosity>1)
299 printf("%s%s: exited with %i (%i)\n", group->prefix, testcase->name, exitcode, status);
300 if (exitcode != 0)
301 {
302 printf("[atexit failure!] ");
303 return FAIL;
304 }
305 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
306 }
307 #endif
308 }
309
310 #endif /* !NO_FORKING */
311
312 int
testcase_run_one(const struct testgroup_t * group,const struct testcase_t * testcase)313 testcase_run_one(const struct testgroup_t *group,
314 const struct testcase_t *testcase)
315 {
316 enum outcome outcome;
317
318 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
319 if (opt_verbosity>0)
320 printf("%s%s: %s\n",
321 group->prefix, testcase->name,
322 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
323 ++n_skipped;
324 return SKIP;
325 }
326
327 if (opt_verbosity>0 && !opt_forked) {
328 printf("%s%s: ", group->prefix, testcase->name);
329 } else {
330 if (opt_verbosity==0) printf(".");
331 cur_test_prefix = group->prefix;
332 cur_test_name = testcase->name;
333 }
334
335 #ifndef NO_FORKING
336 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
337 outcome = testcase_run_forked_(group, testcase);
338 } else {
339 #else
340 {
341 #endif
342 outcome = testcase_run_bare_(testcase);
343 }
344
345 if (outcome == OK) {
346 if (opt_verbosity>0 && !opt_forked)
347 puts(opt_verbosity==1?"OK":"");
348 } else if (outcome == SKIP) {
349 if (opt_verbosity>0 && !opt_forked)
350 puts("SKIPPED");
351 } else {
352 if (!opt_forked)
353 printf("\n [%s FAILED]\n", testcase->name);
354 }
355
356 if (opt_forked) {
357 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
358 return 1; /* unreachable */
359 } else {
360 return (int)outcome;
361 }
362 }
363
364 int
365 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
366 {
367 int i, j;
368 size_t length = LONGEST_TEST_NAME;
369 char fullname[LONGEST_TEST_NAME];
370 int found=0;
371 if (strstr(arg, ".."))
372 length = strstr(arg,"..")-arg;
373 for (i=0; groups[i].prefix; ++i) {
374 for (j=0; groups[i].cases[j].name; ++j) {
375 struct testcase_t *testcase = &groups[i].cases[j];
376 snprintf(fullname, sizeof(fullname), "%s%s",
377 groups[i].prefix, testcase->name);
378 if (!flag) { /* Hack! */
379 printf(" %s", fullname);
380 if (testcase->flags & TT_OFF_BY_DEFAULT)
381 puts(" (Off by default)");
382 else if (testcase->flags & TT_SKIP)
383 puts(" (DISABLED)");
384 else
385 puts("");
386 }
387 if (!strncmp(fullname, arg, length)) {
388 if (set)
389 testcase->flags |= flag;
390 else
391 testcase->flags &= ~flag;
392 ++found;
393 }
394 }
395 }
396 return found;
397 }
398
399 static void
400 usage(struct testgroup_t *groups, int list_groups)
401 {
402 puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
403 puts(" Specify tests by name, or using a prefix ending with '..'");
404 puts(" To skip a test, prefix its name with a colon.");
405 puts(" To enable a disabled test, prefix its name with a plus.");
406 puts(" Use --list-tests for a list of tests.");
407 if (list_groups) {
408 puts("Known tests are:");
409 tinytest_set_flag_(groups, "..", 1, 0);
410 }
411 exit(0);
412 }
413
414 static int
415 process_test_alias(struct testgroup_t *groups, const char *test)
416 {
417 int i, j, n, r;
418 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
419 if (!strcmp(cfg_aliases[i].name, test)) {
420 n = 0;
421 for (j = 0; cfg_aliases[i].tests[j]; ++j) {
422 r = process_test_option(groups, cfg_aliases[i].tests[j]);
423 if (r<0)
424 return -1;
425 n += r;
426 }
427 return n;
428 }
429 }
430 printf("No such test alias as @%s!",test);
431 return -1;
432 }
433
434 static int
435 process_test_option(struct testgroup_t *groups, const char *test)
436 {
437 int flag = TT_ENABLED_;
438 int n = 0;
439 if (test[0] == '@') {
440 return process_test_alias(groups, test + 1);
441 } else if (test[0] == ':') {
442 ++test;
443 flag = TT_SKIP;
444 } else if (test[0] == '+') {
445 ++test;
446 ++n;
447 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
448 printf("No such test as %s!\n", test);
449 return -1;
450 }
451 } else {
452 ++n;
453 }
454 if (!tinytest_set_flag_(groups, test, 1, flag)) {
455 printf("No such test as %s!\n", test);
456 return -1;
457 }
458 return n;
459 }
460
461 void
462 tinytest_set_aliases(const struct testlist_alias_t *aliases)
463 {
464 cfg_aliases = aliases;
465 }
466
467 int
468 tinytest_main(int c, const char **v, struct testgroup_t *groups)
469 {
470 int i, j, n=0;
471
472 #ifdef _WIN32
473 const char *sp = strrchr(v[0], '.');
474 const char *extension = "";
475 if (!sp || stricmp(sp, ".exe"))
476 extension = ".exe"; /* Add an exe so CreateProcess will work */
477 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
478 commandname[MAX_PATH]='\0';
479 #endif
480 for (i=1; i<c; ++i) {
481 if (v[i][0] == '-') {
482 if (!strcmp(v[i], "--RUNNING-FORKED")) {
483 opt_forked = 1;
484 } else if (!strcmp(v[i], "--no-fork")) {
485 opt_nofork = 1;
486 } else if (!strcmp(v[i], "--quiet")) {
487 opt_verbosity = -1;
488 verbosity_flag = "--quiet";
489 } else if (!strcmp(v[i], "--verbose")) {
490 opt_verbosity = 2;
491 verbosity_flag = "--verbose";
492 } else if (!strcmp(v[i], "--terse")) {
493 opt_verbosity = 0;
494 verbosity_flag = "--terse";
495 } else if (!strcmp(v[i], "--help")) {
496 usage(groups, 0);
497 } else if (!strcmp(v[i], "--list-tests")) {
498 usage(groups, 1);
499 } else if (!strcmp(v[i], "--timeout")) {
500 ++i;
501 if (i >= c) {
502 fprintf(stderr, "--timeout requires argument\n");
503 return -1;
504 }
505 opt_timeout = (unsigned)atoi(v[i]);
506 } else {
507 fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
508 return -1;
509 }
510 } else {
511 int r = process_test_option(groups, v[i]);
512 if (r<0)
513 return -1;
514 n += r;
515 }
516 }
517 if (!n)
518 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
519
520 #ifdef _IONBF
521 setvbuf(stdout, NULL, _IONBF, 0);
522 #endif
523
524 ++in_tinytest_main;
525 for (i = 0; groups[i].prefix; ++i) {
526 struct testgroup_t *group = &groups[i];
527 for (j = 0; group->cases[j].name; ++j) {
528 struct testcase_t *testcase = &group->cases[j];
529 int test_attempts = 3;
530 int test_ret_err;
531
532 if (!(testcase->flags & TT_ENABLED_))
533 continue;
534
535 for (;;) {
536 test_ret_err = testcase_run_one(group, testcase);
537
538 if (test_ret_err == OK)
539 break;
540 if (!(testcase->flags & TT_RETRIABLE))
541 break;
542 printf("\n [RETRYING %s (%i)]\n", testcase->name, test_attempts);
543 if (!test_attempts--)
544 break;
545 }
546
547 switch (test_ret_err) {
548 case OK: ++n_ok; break;
549 case SKIP: ++n_skipped; break;
550 default: ++n_bad; break;
551 }
552 }
553 }
554
555 --in_tinytest_main;
556
557 if (opt_verbosity==0)
558 puts("");
559
560 if (n_bad)
561 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
562 n_bad+n_ok,n_skipped);
563 else if (opt_verbosity >= 1)
564 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
565
566 return (n_bad == 0) ? 0 : 1;
567 }
568
569 int
570 tinytest_get_verbosity_(void)
571 {
572 return opt_verbosity;
573 }
574
575 void
576 tinytest_set_test_failed_(void)
577 {
578 if (opt_verbosity <= 0 && cur_test_name) {
579 if (opt_verbosity==0) puts("");
580 printf("%s%s: ", cur_test_prefix, cur_test_name);
581 cur_test_name = NULL;
582 }
583 cur_test_outcome = FAIL;
584 }
585
586 void
587 tinytest_set_test_skipped_(void)
588 {
589 if (cur_test_outcome==OK)
590 cur_test_outcome = SKIP;
591 }
592
593 char *
594 tinytest_format_hex_(const void *val_, unsigned long len)
595 {
596 const unsigned char *val = val_;
597 char *result, *cp;
598 size_t i;
599
600 if (!val)
601 return strdup("null");
602 if (!(result = malloc(len*2+1)))
603 return strdup("<allocation failure>");
604 cp = result;
605 for (i=0;i<len;++i) {
606 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
607 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
608 }
609 *cp = 0;
610 return result;
611 }
612