1 /* $NetBSD: tinytest.c,v 1.1.1.1 2013/12/27 23:31:29 christos Exp $ */ 2 3 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 #ifdef TINYTEST_LOCAL 28 #include "tinytest_local.h" 29 #endif 30 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <assert.h> 35 36 #ifdef _WIN32 37 #include <windows.h> 38 #else 39 #include <sys/types.h> 40 #include <sys/wait.h> 41 #include <unistd.h> 42 #endif 43 44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 46 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 47 /* Workaround for a stupid bug in OSX 10.6 */ 48 #define FORK_BREAKS_GCOV 49 #include <vproc.h> 50 #endif 51 #endif 52 53 #ifndef __GNUC__ 54 #define __attribute__(x) 55 #endif 56 57 #include "tinytest.h" 58 #include "tinytest_macros.h" 59 60 #define LONGEST_TEST_NAME 16384 61 62 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 63 static int n_ok = 0; /**< Number of tests that have passed */ 64 static int n_bad = 0; /**< Number of tests that have failed. */ 65 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 66 67 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 68 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 69 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 70 const char *verbosity_flag = ""; 71 72 const struct testlist_alias_t *cfg_aliases=NULL; 73 74 enum outcome { SKIP=2, OK=1, FAIL=0 }; 75 static enum outcome cur_test_outcome = 0; 76 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 77 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 78 const char *cur_test_name = NULL; 79 80 #ifdef _WIN32 81 /* Copy of argv[0] for win32. */ 82 static char commandname[MAX_PATH+1]; 83 #endif 84 85 static void usage(struct testgroup_t *groups, int list_groups) 86 __attribute__((noreturn)); 87 static int process_test_option(struct testgroup_t *groups, const char *test); 88 89 static enum outcome 90 testcase_run_bare_(const struct testcase_t *testcase) 91 { 92 void *env = NULL; 93 int outcome; 94 if (testcase->setup) { 95 env = testcase->setup->setup_fn(testcase); 96 if (!env) 97 return FAIL; 98 else if (env == (void*)TT_SKIP) 99 return SKIP; 100 } 101 102 cur_test_outcome = OK; 103 testcase->fn(env); 104 outcome = cur_test_outcome; 105 106 if (testcase->setup) { 107 if (testcase->setup->cleanup_fn(testcase, env) == 0) 108 outcome = FAIL; 109 } 110 111 return outcome; 112 } 113 114 #define MAGIC_EXITCODE 42 115 116 static enum outcome 117 testcase_run_forked_(const struct testgroup_t *group, 118 const struct testcase_t *testcase) 119 { 120 #ifdef _WIN32 121 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 122 we'll invoke our own exe (whose name we recall from the command 123 line) with a command line that tells it to run just the test we 124 want, and this time without forking. 125 126 (No, threads aren't an option. The whole point of forking is to 127 share no state between tests.) 128 */ 129 int ok; 130 char buffer[LONGEST_TEST_NAME+256]; 131 STARTUPINFOA si; 132 PROCESS_INFORMATION info; 133 DWORD exitcode; 134 135 if (!in_tinytest_main) { 136 printf("\nERROR. On Windows, testcase_run_forked_ must be" 137 " called from within tinytest_main.\n"); 138 abort(); 139 } 140 if (opt_verbosity>0) 141 printf("[forking] "); 142 143 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 144 commandname, verbosity_flag, group->prefix, testcase->name); 145 146 memset(&si, 0, sizeof(si)); 147 memset(&info, 0, sizeof(info)); 148 si.cb = sizeof(si); 149 150 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 151 0, NULL, NULL, &si, &info); 152 if (!ok) { 153 printf("CreateProcess failed!\n"); 154 return 0; 155 } 156 WaitForSingleObject(info.hProcess, INFINITE); 157 GetExitCodeProcess(info.hProcess, &exitcode); 158 CloseHandle(info.hProcess); 159 CloseHandle(info.hThread); 160 if (exitcode == 0) 161 return OK; 162 else if (exitcode == MAGIC_EXITCODE) 163 return SKIP; 164 else 165 return FAIL; 166 #else 167 int outcome_pipe[2]; 168 pid_t pid; 169 (void)group; 170 171 if (pipe(outcome_pipe)) 172 perror("opening pipe"); 173 174 if (opt_verbosity>0) 175 printf("[forking] "); 176 pid = fork(); 177 #ifdef FORK_BREAKS_GCOV 178 vproc_transaction_begin(0); 179 #endif 180 if (!pid) { 181 /* child. */ 182 int test_r, write_r; 183 char b[1]; 184 close(outcome_pipe[0]); 185 test_r = testcase_run_bare_(testcase); 186 assert(0<=(int)test_r && (int)test_r<=2); 187 b[0] = "NYS"[test_r]; 188 write_r = (int)write(outcome_pipe[1], b, 1); 189 if (write_r != 1) { 190 perror("write outcome to pipe"); 191 exit(1); 192 } 193 exit(0); 194 return FAIL; /* unreachable */ 195 } else { 196 /* parent */ 197 int status, r; 198 char b[1]; 199 /* Close this now, so that if the other side closes it, 200 * our read fails. */ 201 close(outcome_pipe[1]); 202 r = (int)read(outcome_pipe[0], b, 1); 203 if (r == 0) { 204 printf("[Lost connection!] "); 205 return 0; 206 } else if (r != 1) { 207 perror("read outcome from pipe"); 208 } 209 waitpid(pid, &status, 0); 210 close(outcome_pipe[0]); 211 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 212 } 213 #endif 214 } 215 216 int 217 testcase_run_one(const struct testgroup_t *group, 218 const struct testcase_t *testcase) 219 { 220 enum outcome outcome; 221 222 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 223 if (opt_verbosity>0) 224 printf("%s%s: %s\n", 225 group->prefix, testcase->name, 226 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 227 ++n_skipped; 228 return SKIP; 229 } 230 231 if (opt_verbosity>0 && !opt_forked) { 232 printf("%s%s: ", group->prefix, testcase->name); 233 } else { 234 if (opt_verbosity==0) printf("."); 235 cur_test_prefix = group->prefix; 236 cur_test_name = testcase->name; 237 } 238 239 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 240 outcome = testcase_run_forked_(group, testcase); 241 } else { 242 outcome = testcase_run_bare_(testcase); 243 } 244 245 if (outcome == OK) { 246 ++n_ok; 247 if (opt_verbosity>0 && !opt_forked) 248 puts(opt_verbosity==1?"OK":""); 249 } else if (outcome == SKIP) { 250 ++n_skipped; 251 if (opt_verbosity>0 && !opt_forked) 252 puts("SKIPPED"); 253 } else { 254 ++n_bad; 255 if (!opt_forked) 256 printf("\n [%s FAILED]\n", testcase->name); 257 } 258 259 if (opt_forked) { 260 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 261 return 1; /* unreachable */ 262 } else { 263 return (int)outcome; 264 } 265 } 266 267 int 268 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 269 { 270 int i, j; 271 size_t length = LONGEST_TEST_NAME; 272 char fullname[LONGEST_TEST_NAME]; 273 int found=0; 274 if (strstr(arg, "..")) 275 length = strstr(arg,"..")-arg; 276 for (i=0; groups[i].prefix; ++i) { 277 for (j=0; groups[i].cases[j].name; ++j) { 278 struct testcase_t *testcase = &groups[i].cases[j]; 279 snprintf(fullname, sizeof(fullname), "%s%s", 280 groups[i].prefix, testcase->name); 281 if (!flag) { /* Hack! */ 282 printf(" %s", fullname); 283 if (testcase->flags & TT_OFF_BY_DEFAULT) 284 puts(" (Off by default)"); 285 else if (testcase->flags & TT_SKIP) 286 puts(" (DISABLED)"); 287 else 288 puts(""); 289 } 290 if (!strncmp(fullname, arg, length)) { 291 if (set) 292 testcase->flags |= flag; 293 else 294 testcase->flags &= ~flag; 295 ++found; 296 } 297 } 298 } 299 return found; 300 } 301 302 static void 303 usage(struct testgroup_t *groups, int list_groups) 304 { 305 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 306 puts(" Specify tests by name, or using a prefix ending with '..'"); 307 puts(" To skip a test, prefix its name with a colon."); 308 puts(" To enable a disabled test, prefix its name with a plus."); 309 puts(" Use --list-tests for a list of tests."); 310 if (list_groups) { 311 puts("Known tests are:"); 312 tinytest_set_flag_(groups, "..", 1, 0); 313 } 314 exit(0); 315 } 316 317 static int 318 process_test_alias(struct testgroup_t *groups, const char *test) 319 { 320 int i, j, n, r; 321 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 322 if (!strcmp(cfg_aliases[i].name, test)) { 323 n = 0; 324 for (j = 0; cfg_aliases[i].tests[j]; ++j) { 325 r = process_test_option(groups, cfg_aliases[i].tests[j]); 326 if (r<0) 327 return -1; 328 n += r; 329 } 330 return n; 331 } 332 } 333 printf("No such test alias as @%s!",test); 334 return -1; 335 } 336 337 static int 338 process_test_option(struct testgroup_t *groups, const char *test) 339 { 340 int flag = TT_ENABLED_; 341 int n = 0; 342 if (test[0] == '@') { 343 return process_test_alias(groups, test + 1); 344 } else if (test[0] == ':') { 345 ++test; 346 flag = TT_SKIP; 347 } else if (test[0] == '+') { 348 ++test; 349 ++n; 350 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 351 printf("No such test as %s!\n", test); 352 return -1; 353 } 354 } else { 355 ++n; 356 } 357 if (!tinytest_set_flag_(groups, test, 1, flag)) { 358 printf("No such test as %s!\n", test); 359 return -1; 360 } 361 return n; 362 } 363 364 void 365 tinytest_set_aliases(const struct testlist_alias_t *aliases) 366 { 367 cfg_aliases = aliases; 368 } 369 370 int 371 tinytest_main(int c, const char **v, struct testgroup_t *groups) 372 { 373 int i, j, n=0; 374 375 #ifdef _WIN32 376 const char *sp = strrchr(v[0], '.'); 377 const char *extension = ""; 378 if (!sp || stricmp(sp, ".exe")) 379 extension = ".exe"; /* Add an exe so CreateProcess will work */ 380 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 381 commandname[MAX_PATH]='\0'; 382 #endif 383 for (i=1; i<c; ++i) { 384 if (v[i][0] == '-') { 385 if (!strcmp(v[i], "--RUNNING-FORKED")) { 386 opt_forked = 1; 387 } else if (!strcmp(v[i], "--no-fork")) { 388 opt_nofork = 1; 389 } else if (!strcmp(v[i], "--quiet")) { 390 opt_verbosity = -1; 391 verbosity_flag = "--quiet"; 392 } else if (!strcmp(v[i], "--verbose")) { 393 opt_verbosity = 2; 394 verbosity_flag = "--verbose"; 395 } else if (!strcmp(v[i], "--terse")) { 396 opt_verbosity = 0; 397 verbosity_flag = "--terse"; 398 } else if (!strcmp(v[i], "--help")) { 399 usage(groups, 0); 400 } else if (!strcmp(v[i], "--list-tests")) { 401 usage(groups, 1); 402 } else { 403 printf("Unknown option %s. Try --help\n",v[i]); 404 return -1; 405 } 406 } else { 407 int r = process_test_option(groups, v[i]); 408 if (r<0) 409 return -1; 410 n += r; 411 } 412 } 413 if (!n) 414 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 415 416 setvbuf(stdout, NULL, _IONBF, 0); 417 418 ++in_tinytest_main; 419 for (i=0; groups[i].prefix; ++i) 420 for (j=0; groups[i].cases[j].name; ++j) 421 if (groups[i].cases[j].flags & TT_ENABLED_) 422 testcase_run_one(&groups[i], 423 &groups[i].cases[j]); 424 425 --in_tinytest_main; 426 427 if (opt_verbosity==0) 428 puts(""); 429 430 if (n_bad) 431 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 432 n_bad+n_ok,n_skipped); 433 else if (opt_verbosity >= 1) 434 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 435 436 return (n_bad == 0) ? 0 : 1; 437 } 438 439 int 440 tinytest_get_verbosity_(void) 441 { 442 return opt_verbosity; 443 } 444 445 void 446 tinytest_set_test_failed_(void) 447 { 448 if (opt_verbosity <= 0 && cur_test_name) { 449 if (opt_verbosity==0) puts(""); 450 printf("%s%s: ", cur_test_prefix, cur_test_name); 451 cur_test_name = NULL; 452 } 453 cur_test_outcome = 0; 454 } 455 456 void 457 tinytest_set_test_skipped_(void) 458 { 459 if (cur_test_outcome==OK) 460 cur_test_outcome = SKIP; 461 } 462 463