1 /* $NetBSD: tinytest.c,v 1.1.1.2 2017/01/31 21:14:53 christos Exp $ */ 2 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. The name of the author may not be used to endorse or promote products 13 * derived from this software without specific prior written permission. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 #ifdef TINYTEST_LOCAL 27 #include "tinytest_local.h" 28 #endif 29 30 #include <stdio.h> 31 #include <stdlib.h> 32 #include <string.h> 33 #include <assert.h> 34 35 #ifndef NO_FORKING 36 37 #ifdef _WIN32 38 #include <windows.h> 39 #else 40 #include <sys/types.h> 41 #include <sys/wait.h> 42 #include <unistd.h> 43 #endif 44 45 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 46 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 47 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 48 /* Workaround for a stupid bug in OSX 10.6 */ 49 #define FORK_BREAKS_GCOV 50 #include <vproc.h> 51 #endif 52 #endif 53 54 #endif /* !NO_FORKING */ 55 56 #ifndef __GNUC__ 57 #define __attribute__(x) 58 #endif 59 60 #include "tinytest.h" 61 #include "tinytest_macros.h" 62 63 #define LONGEST_TEST_NAME 16384 64 65 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 66 static int n_ok = 0; /**< Number of tests that have passed */ 67 static int n_bad = 0; /**< Number of tests that have failed. */ 68 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 69 70 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 71 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 72 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 73 const char *verbosity_flag = ""; 74 75 const struct testlist_alias_t *cfg_aliases=NULL; 76 77 enum outcome { SKIP=2, OK=1, FAIL=0 }; 78 static enum outcome cur_test_outcome = 0; 79 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 80 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 81 const char *cur_test_name = NULL; 82 83 #ifdef _WIN32 84 /* Copy of argv[0] for win32. */ 85 static char commandname[MAX_PATH+1]; 86 #endif 87 88 static void usage(struct testgroup_t *groups, int list_groups) 89 __attribute__((noreturn)); 90 static int process_test_option(struct testgroup_t *groups, const char *test); 91 92 static enum outcome 93 testcase_run_bare_(const struct testcase_t *testcase) 94 { 95 void *env = NULL; 96 int outcome; 97 if (testcase->setup) { 98 env = testcase->setup->setup_fn(testcase); 99 if (!env) 100 return FAIL; 101 else if (env == (void*)TT_SKIP) 102 return SKIP; 103 } 104 105 cur_test_outcome = OK; 106 testcase->fn(env); 107 outcome = cur_test_outcome; 108 109 if (testcase->setup) { 110 if (testcase->setup->cleanup_fn(testcase, env) == 0) 111 outcome = FAIL; 112 } 113 114 return outcome; 115 } 116 117 #define MAGIC_EXITCODE 42 118 119 #ifndef NO_FORKING 120 121 static enum outcome 122 testcase_run_forked_(const struct testgroup_t *group, 123 const struct testcase_t *testcase) 124 { 125 #ifdef _WIN32 126 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 127 we'll invoke our own exe (whose name we recall from the command 128 line) with a command line that tells it to run just the test we 129 want, and this time without forking. 130 131 (No, threads aren't an option. The whole point of forking is to 132 share no state between tests.) 133 */ 134 int ok; 135 char buffer[LONGEST_TEST_NAME+256]; 136 STARTUPINFOA si; 137 PROCESS_INFORMATION info; 138 DWORD exitcode; 139 140 if (!in_tinytest_main) { 141 printf("\nERROR. On Windows, testcase_run_forked_ must be" 142 " called from within tinytest_main.\n"); 143 abort(); 144 } 145 if (opt_verbosity>0) 146 printf("[forking] "); 147 148 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 149 commandname, verbosity_flag, group->prefix, testcase->name); 150 151 memset(&si, 0, sizeof(si)); 152 memset(&info, 0, sizeof(info)); 153 si.cb = sizeof(si); 154 155 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 156 0, NULL, NULL, &si, &info); 157 if (!ok) { 158 printf("CreateProcess failed!\n"); 159 return 0; 160 } 161 WaitForSingleObject(info.hProcess, INFINITE); 162 GetExitCodeProcess(info.hProcess, &exitcode); 163 CloseHandle(info.hProcess); 164 CloseHandle(info.hThread); 165 if (exitcode == 0) 166 return OK; 167 else if (exitcode == MAGIC_EXITCODE) 168 return SKIP; 169 else 170 return FAIL; 171 #else 172 int outcome_pipe[2]; 173 pid_t pid; 174 (void)group; 175 176 if (pipe(outcome_pipe)) 177 perror("opening pipe"); 178 179 if (opt_verbosity>0) 180 printf("[forking] "); 181 pid = fork(); 182 #ifdef FORK_BREAKS_GCOV 183 vproc_transaction_begin(0); 184 #endif 185 if (!pid) { 186 /* child. */ 187 int test_r, write_r; 188 char b[1]; 189 close(outcome_pipe[0]); 190 test_r = testcase_run_bare_(testcase); 191 assert(0<=(int)test_r && (int)test_r<=2); 192 b[0] = "NYS"[test_r]; 193 write_r = (int)write(outcome_pipe[1], b, 1); 194 if (write_r != 1) { 195 perror("write outcome to pipe"); 196 exit(1); 197 } 198 exit(0); 199 return FAIL; /* unreachable */ 200 } else { 201 /* parent */ 202 int status, r; 203 char b[1]; 204 /* Close this now, so that if the other side closes it, 205 * our read fails. */ 206 close(outcome_pipe[1]); 207 r = (int)read(outcome_pipe[0], b, 1); 208 if (r == 0) { 209 printf("[Lost connection!] "); 210 return 0; 211 } else if (r != 1) { 212 perror("read outcome from pipe"); 213 } 214 waitpid(pid, &status, 0); 215 close(outcome_pipe[0]); 216 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 217 } 218 #endif 219 } 220 221 #endif /* !NO_FORKING */ 222 223 int 224 testcase_run_one(const struct testgroup_t *group, 225 const struct testcase_t *testcase) 226 { 227 enum outcome outcome; 228 229 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 230 if (opt_verbosity>0) 231 printf("%s%s: %s\n", 232 group->prefix, testcase->name, 233 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 234 ++n_skipped; 235 return SKIP; 236 } 237 238 if (opt_verbosity>0 && !opt_forked) { 239 printf("%s%s: ", group->prefix, testcase->name); 240 } else { 241 if (opt_verbosity==0) printf("."); 242 cur_test_prefix = group->prefix; 243 cur_test_name = testcase->name; 244 } 245 246 #ifndef NO_FORKING 247 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 248 outcome = testcase_run_forked_(group, testcase); 249 } else { 250 #else 251 { 252 #endif 253 outcome = testcase_run_bare_(testcase); 254 } 255 256 if (outcome == OK) { 257 ++n_ok; 258 if (opt_verbosity>0 && !opt_forked) 259 puts(opt_verbosity==1?"OK":""); 260 } else if (outcome == SKIP) { 261 ++n_skipped; 262 if (opt_verbosity>0 && !opt_forked) 263 puts("SKIPPED"); 264 } else { 265 ++n_bad; 266 if (!opt_forked) 267 printf("\n [%s FAILED]\n", testcase->name); 268 } 269 270 if (opt_forked) { 271 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 272 return 1; /* unreachable */ 273 } else { 274 return (int)outcome; 275 } 276 } 277 278 int 279 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 280 { 281 int i, j; 282 size_t length = LONGEST_TEST_NAME; 283 char fullname[LONGEST_TEST_NAME]; 284 int found=0; 285 if (strstr(arg, "..")) 286 length = strstr(arg,"..")-arg; 287 for (i=0; groups[i].prefix; ++i) { 288 for (j=0; groups[i].cases[j].name; ++j) { 289 struct testcase_t *testcase = &groups[i].cases[j]; 290 snprintf(fullname, sizeof(fullname), "%s%s", 291 groups[i].prefix, testcase->name); 292 if (!flag) { /* Hack! */ 293 printf(" %s", fullname); 294 if (testcase->flags & TT_OFF_BY_DEFAULT) 295 puts(" (Off by default)"); 296 else if (testcase->flags & TT_SKIP) 297 puts(" (DISABLED)"); 298 else 299 puts(""); 300 } 301 if (!strncmp(fullname, arg, length)) { 302 if (set) 303 testcase->flags |= flag; 304 else 305 testcase->flags &= ~flag; 306 ++found; 307 } 308 } 309 } 310 return found; 311 } 312 313 static void 314 usage(struct testgroup_t *groups, int list_groups) 315 { 316 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 317 puts(" Specify tests by name, or using a prefix ending with '..'"); 318 puts(" To skip a test, prefix its name with a colon."); 319 puts(" To enable a disabled test, prefix its name with a plus."); 320 puts(" Use --list-tests for a list of tests."); 321 if (list_groups) { 322 puts("Known tests are:"); 323 tinytest_set_flag_(groups, "..", 1, 0); 324 } 325 exit(0); 326 } 327 328 static int 329 process_test_alias(struct testgroup_t *groups, const char *test) 330 { 331 int i, j, n, r; 332 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 333 if (!strcmp(cfg_aliases[i].name, test)) { 334 n = 0; 335 for (j = 0; cfg_aliases[i].tests[j]; ++j) { 336 r = process_test_option(groups, cfg_aliases[i].tests[j]); 337 if (r<0) 338 return -1; 339 n += r; 340 } 341 return n; 342 } 343 } 344 printf("No such test alias as @%s!",test); 345 return -1; 346 } 347 348 static int 349 process_test_option(struct testgroup_t *groups, const char *test) 350 { 351 int flag = TT_ENABLED_; 352 int n = 0; 353 if (test[0] == '@') { 354 return process_test_alias(groups, test + 1); 355 } else if (test[0] == ':') { 356 ++test; 357 flag = TT_SKIP; 358 } else if (test[0] == '+') { 359 ++test; 360 ++n; 361 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 362 printf("No such test as %s!\n", test); 363 return -1; 364 } 365 } else { 366 ++n; 367 } 368 if (!tinytest_set_flag_(groups, test, 1, flag)) { 369 printf("No such test as %s!\n", test); 370 return -1; 371 } 372 return n; 373 } 374 375 void 376 tinytest_set_aliases(const struct testlist_alias_t *aliases) 377 { 378 cfg_aliases = aliases; 379 } 380 381 int 382 tinytest_main(int c, const char **v, struct testgroup_t *groups) 383 { 384 int i, j, n=0; 385 386 #ifdef _WIN32 387 const char *sp = strrchr(v[0], '.'); 388 const char *extension = ""; 389 if (!sp || stricmp(sp, ".exe")) 390 extension = ".exe"; /* Add an exe so CreateProcess will work */ 391 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 392 commandname[MAX_PATH]='\0'; 393 #endif 394 for (i=1; i<c; ++i) { 395 if (v[i][0] == '-') { 396 if (!strcmp(v[i], "--RUNNING-FORKED")) { 397 opt_forked = 1; 398 } else if (!strcmp(v[i], "--no-fork")) { 399 opt_nofork = 1; 400 } else if (!strcmp(v[i], "--quiet")) { 401 opt_verbosity = -1; 402 verbosity_flag = "--quiet"; 403 } else if (!strcmp(v[i], "--verbose")) { 404 opt_verbosity = 2; 405 verbosity_flag = "--verbose"; 406 } else if (!strcmp(v[i], "--terse")) { 407 opt_verbosity = 0; 408 verbosity_flag = "--terse"; 409 } else if (!strcmp(v[i], "--help")) { 410 usage(groups, 0); 411 } else if (!strcmp(v[i], "--list-tests")) { 412 usage(groups, 1); 413 } else { 414 printf("Unknown option %s. Try --help\n",v[i]); 415 return -1; 416 } 417 } else { 418 int r = process_test_option(groups, v[i]); 419 if (r<0) 420 return -1; 421 n += r; 422 } 423 } 424 if (!n) 425 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 426 427 #ifdef _IONBF 428 setvbuf(stdout, NULL, _IONBF, 0); 429 #endif 430 431 ++in_tinytest_main; 432 for (i=0; groups[i].prefix; ++i) 433 for (j=0; groups[i].cases[j].name; ++j) 434 if (groups[i].cases[j].flags & TT_ENABLED_) 435 testcase_run_one(&groups[i], 436 &groups[i].cases[j]); 437 438 --in_tinytest_main; 439 440 if (opt_verbosity==0) 441 puts(""); 442 443 if (n_bad) 444 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 445 n_bad+n_ok,n_skipped); 446 else if (opt_verbosity >= 1) 447 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 448 449 return (n_bad == 0) ? 0 : 1; 450 } 451 452 int 453 tinytest_get_verbosity_(void) 454 { 455 return opt_verbosity; 456 } 457 458 void 459 tinytest_set_test_failed_(void) 460 { 461 if (opt_verbosity <= 0 && cur_test_name) { 462 if (opt_verbosity==0) puts(""); 463 printf("%s%s: ", cur_test_prefix, cur_test_name); 464 cur_test_name = NULL; 465 } 466 cur_test_outcome = 0; 467 } 468 469 void 470 tinytest_set_test_skipped_(void) 471 { 472 if (cur_test_outcome==OK) 473 cur_test_outcome = SKIP; 474 } 475 476 char * 477 tinytest_format_hex_(const void *val_, unsigned long len) 478 { 479 const unsigned char *val = val_; 480 char *result, *cp; 481 size_t i; 482 483 if (!val) 484 return strdup("null"); 485 if (!(result = malloc(len*2+1))) 486 return strdup("<allocation failure>"); 487 cp = result; 488 for (i=0;i<len;++i) { 489 *cp++ = "0123456789ABCDEF"[val[i] >> 4]; 490 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; 491 } 492 *cp = 0; 493 return result; 494 } 495