1 /* $NetBSD: tinytest.c,v 1.7 2024/08/18 20:47:24 christos Exp $ */ 2 3 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 #ifdef TINYTEST_LOCAL 28 #include "tinytest_local.h" 29 #endif 30 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <assert.h> 35 36 #ifndef NO_FORKING 37 38 #ifdef _WIN32 39 #include <windows.h> 40 #else 41 #include <sys/types.h> 42 #include <sys/wait.h> 43 #include <unistd.h> 44 #endif 45 46 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 47 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 48 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 49 /* Workaround for a stupid bug in OSX 10.6 */ 50 #define FORK_BREAKS_GCOV 51 #include <vproc.h> 52 #endif 53 #endif 54 55 #endif /* !NO_FORKING */ 56 57 #ifndef __GNUC__ 58 #define __attribute__(x) 59 #endif 60 61 #include "tinytest.h" 62 #include "tinytest_macros.h" 63 64 #define LONGEST_TEST_NAME 16384 65 #define DEFAULT_TESTCASE_TIMEOUT 30U 66 #define MAGIC_EXITCODE 42 67 68 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 69 static int n_ok = 0; /**< Number of tests that have passed */ 70 static int n_bad = 0; /**< Number of tests that have failed. */ 71 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 72 73 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 74 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 75 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 76 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */ 77 const char *verbosity_flag = ""; 78 79 const struct testlist_alias_t *cfg_aliases=NULL; 80 81 enum outcome { SKIP=2, OK=1, FAIL=0 }; 82 static enum outcome cur_test_outcome = 0; 83 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 84 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 85 const char *cur_test_name = NULL; 86 87 static void usage(struct testgroup_t *groups, int list_groups) 88 __attribute__((noreturn)); 89 static int process_test_option(struct testgroup_t *groups, const char *test); 90 91 #ifdef _WIN32 92 /* Copy of argv[0] for win32. */ 93 static char commandname[MAX_PATH+1]; 94 95 struct timeout_thread_args { 96 const testcase_fn *fn; 97 void *env; 98 }; 99 100 static DWORD WINAPI 101 timeout_thread_proc_(LPVOID arg) 102 { 103 struct timeout_thread_args *args = arg; 104 (*(args->fn))(args->env); 105 ExitThread(cur_test_outcome == FAIL ? 1 : 0); 106 } 107 108 static enum outcome 109 testcase_run_in_thread_(const struct testcase_t *testcase, void *env) 110 { 111 /* We will never run testcase in a new thread when the 112 timeout is set to zero */ 113 assert(opt_timeout); 114 DWORD ret, tid; 115 HANDLE handle; 116 struct timeout_thread_args args = { 117 &(testcase->fn), 118 env 119 }; 120 121 handle =CreateThread(NULL, 0, timeout_thread_proc_, 122 (LPVOID)&args, 0, &tid); 123 ret = WaitForSingleObject(handle, opt_timeout * 1000U); 124 if (ret == WAIT_OBJECT_0) { 125 ret = 0; 126 if (!GetExitCodeThread(handle, &ret)) { 127 printf("GetExitCodeThread failed\n"); 128 ret = 1; 129 } 130 } else if (ret == WAIT_TIMEOUT) { 131 printf("timeout\n"); 132 } else { 133 printf("Wait failed\n"); 134 } 135 CloseHandle(handle); 136 if (ret == 0) 137 return OK; 138 else if (ret == MAGIC_EXITCODE) 139 return SKIP; 140 else 141 return FAIL; 142 } 143 #else 144 static unsigned int testcase_set_timeout_(void) 145 { 146 return alarm(opt_timeout); 147 } 148 149 static unsigned int testcase_reset_timeout_(void) 150 { 151 return alarm(0); 152 } 153 #endif 154 155 static enum outcome 156 testcase_run_bare_(const struct testcase_t *testcase) 157 { 158 void *env = NULL; 159 int outcome; 160 if (testcase->setup) { 161 env = testcase->setup->setup_fn(testcase); 162 if (!env) 163 return FAIL; 164 else if (env == (void*)TT_SKIP) 165 return SKIP; 166 } 167 168 cur_test_outcome = OK; 169 { 170 if (opt_timeout) { 171 #ifdef _WIN32 172 cur_test_outcome = testcase_run_in_thread_(testcase, env); 173 #else 174 testcase_set_timeout_(); 175 testcase->fn(env); 176 testcase_reset_timeout_(); 177 #endif 178 } else { 179 testcase->fn(env); 180 } 181 } 182 outcome = cur_test_outcome; 183 184 if (testcase->setup) { 185 if (testcase->setup->cleanup_fn(testcase, env) == 0) 186 outcome = FAIL; 187 } 188 189 return outcome; 190 } 191 192 193 #ifndef NO_FORKING 194 195 static enum outcome 196 testcase_run_forked_(const struct testgroup_t *group, 197 const struct testcase_t *testcase) 198 { 199 #ifdef _WIN32 200 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 201 we'll invoke our own exe (whose name we recall from the command 202 line) with a command line that tells it to run just the test we 203 want, and this time without forking. 204 205 (No, threads aren't an option. The whole point of forking is to 206 share no state between tests.) 207 */ 208 int ok; 209 char buffer[LONGEST_TEST_NAME+256]; 210 STARTUPINFOA si; 211 PROCESS_INFORMATION info; 212 DWORD ret; 213 214 if (!in_tinytest_main) { 215 printf("\nERROR. On Windows, testcase_run_forked_ must be" 216 " called from within tinytest_main.\n"); 217 abort(); 218 } 219 if (opt_verbosity>0) 220 printf("[forking] "); 221 222 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s --timeout 0 %s%s", 223 commandname, verbosity_flag, group->prefix, testcase->name); 224 225 memset(&si, 0, sizeof(si)); 226 memset(&info, 0, sizeof(info)); 227 si.cb = sizeof(si); 228 229 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 230 0, NULL, NULL, &si, &info); 231 if (!ok) { 232 printf("CreateProcess failed!\n"); 233 return FAIL; 234 } 235 ret = WaitForSingleObject(info.hProcess, 236 (opt_timeout ? opt_timeout * 1000U : INFINITE)); 237 238 if (ret == WAIT_OBJECT_0) { 239 GetExitCodeProcess(info.hProcess, &ret); 240 } else if (ret == WAIT_TIMEOUT) { 241 printf("timeout\n"); 242 } else { 243 printf("Wait failed\n"); 244 } 245 CloseHandle(info.hProcess); 246 CloseHandle(info.hThread); 247 if (ret == 0) 248 return OK; 249 else if (ret == MAGIC_EXITCODE) 250 return SKIP; 251 else 252 return FAIL; 253 #else 254 int outcome_pipe[2]; 255 pid_t pid; 256 (void)group; 257 258 if (pipe(outcome_pipe)) 259 perror("opening pipe"); 260 261 if (opt_verbosity>0) 262 printf("[forking] "); 263 pid = fork(); 264 #ifdef FORK_BREAKS_GCOV 265 vproc_transaction_begin(0); 266 #endif 267 if (!pid) { 268 /* child. */ 269 int test_r, write_r; 270 char b[1]; 271 close(outcome_pipe[0]); 272 test_r = testcase_run_bare_(testcase); 273 assert(0<=(int)test_r && (int)test_r<=2); 274 b[0] = "NYS"[test_r]; 275 write_r = (int)write(outcome_pipe[1], b, 1); 276 if (write_r != 1) { 277 perror("write outcome to pipe"); 278 exit(1); 279 } 280 exit(0); 281 return FAIL; /* unreachable */ 282 } else { 283 /* parent */ 284 int status, r, exitcode; 285 char b[1]; 286 /* Close this now, so that if the other side closes it, 287 * our read fails. */ 288 close(outcome_pipe[1]); 289 r = (int)read(outcome_pipe[0], b, 1); 290 if (r == 0) { 291 printf("[Lost connection!] "); 292 return FAIL; 293 } else if (r != 1) { 294 perror("read outcome from pipe"); 295 } 296 waitpid(pid, &status, 0); 297 exitcode = WEXITSTATUS(status); 298 close(outcome_pipe[0]); 299 if (opt_verbosity>1) 300 printf("%s%s: exited with %i (%i)\n", group->prefix, testcase->name, exitcode, status); 301 if (exitcode != 0) 302 { 303 printf("[atexit failure!] "); 304 return FAIL; 305 } 306 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 307 } 308 #endif 309 } 310 311 #endif /* !NO_FORKING */ 312 313 int 314 testcase_run_one(const struct testgroup_t *group, 315 const struct testcase_t *testcase) 316 { 317 enum outcome outcome; 318 319 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 320 if (opt_verbosity>0) 321 printf("%s%s: %s\n", 322 group->prefix, testcase->name, 323 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 324 ++n_skipped; 325 return SKIP; 326 } 327 328 if (opt_verbosity>0 && !opt_forked) { 329 printf("%s%s: ", group->prefix, testcase->name); 330 } else { 331 if (opt_verbosity==0) printf("."); 332 cur_test_prefix = group->prefix; 333 cur_test_name = testcase->name; 334 } 335 336 #ifndef NO_FORKING 337 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 338 outcome = testcase_run_forked_(group, testcase); 339 } else { 340 #else 341 { 342 #endif 343 outcome = testcase_run_bare_(testcase); 344 } 345 346 if (outcome == OK) { 347 if (opt_verbosity>0 && !opt_forked) 348 puts(opt_verbosity==1?"OK":""); 349 } else if (outcome == SKIP) { 350 if (opt_verbosity>0 && !opt_forked) 351 puts("SKIPPED"); 352 } else { 353 if (!opt_forked) 354 printf("\n [%s FAILED]\n", testcase->name); 355 } 356 357 if (opt_forked) { 358 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 359 return 1; /* unreachable */ 360 } else { 361 return (int)outcome; 362 } 363 } 364 365 int 366 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 367 { 368 int i, j; 369 size_t length = LONGEST_TEST_NAME; 370 char fullname[LONGEST_TEST_NAME]; 371 int found=0; 372 if (strstr(arg, "..")) 373 length = strstr(arg,"..")-arg; 374 for (i=0; groups[i].prefix; ++i) { 375 for (j=0; groups[i].cases[j].name; ++j) { 376 struct testcase_t *testcase = &groups[i].cases[j]; 377 snprintf(fullname, sizeof(fullname), "%s%s", 378 groups[i].prefix, testcase->name); 379 if (!flag) { /* Hack! */ 380 printf(" %s", fullname); 381 if (testcase->flags & TT_OFF_BY_DEFAULT) 382 puts(" (Off by default)"); 383 else if (testcase->flags & TT_SKIP) 384 puts(" (DISABLED)"); 385 else 386 puts(""); 387 } 388 if (!strncmp(fullname, arg, length)) { 389 if (set) 390 testcase->flags |= flag; 391 else 392 testcase->flags &= ~flag; 393 ++found; 394 } 395 } 396 } 397 return found; 398 } 399 400 static void 401 usage(struct testgroup_t *groups, int list_groups) 402 { 403 puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]"); 404 puts(" Specify tests by name, or using a prefix ending with '..'"); 405 puts(" To skip a test, prefix its name with a colon."); 406 puts(" To enable a disabled test, prefix its name with a plus."); 407 puts(" Use --list-tests for a list of tests."); 408 if (list_groups) { 409 puts("Known tests are:"); 410 tinytest_set_flag_(groups, "..", 1, 0); 411 } 412 exit(0); 413 } 414 415 static int 416 process_test_alias(struct testgroup_t *groups, const char *test) 417 { 418 int i, j, n, r; 419 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 420 if (!strcmp(cfg_aliases[i].name, test)) { 421 n = 0; 422 for (j = 0; cfg_aliases[i].tests[j]; ++j) { 423 r = process_test_option(groups, cfg_aliases[i].tests[j]); 424 if (r<0) 425 return -1; 426 n += r; 427 } 428 return n; 429 } 430 } 431 printf("No such test alias as @%s!",test); 432 return -1; 433 } 434 435 static int 436 process_test_option(struct testgroup_t *groups, const char *test) 437 { 438 int flag = TT_ENABLED_; 439 int n = 0; 440 if (test[0] == '@') { 441 return process_test_alias(groups, test + 1); 442 } else if (test[0] == ':') { 443 ++test; 444 flag = TT_SKIP; 445 } else if (test[0] == '+') { 446 ++test; 447 ++n; 448 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 449 printf("No such test as %s!\n", test); 450 return -1; 451 } 452 } else { 453 ++n; 454 } 455 if (!tinytest_set_flag_(groups, test, 1, flag)) { 456 printf("No such test as %s!\n", test); 457 return -1; 458 } 459 return n; 460 } 461 462 void 463 tinytest_set_aliases(const struct testlist_alias_t *aliases) 464 { 465 cfg_aliases = aliases; 466 } 467 468 int 469 tinytest_main(int c, const char **v, struct testgroup_t *groups) 470 { 471 int i, j, n=0; 472 473 #ifdef _WIN32 474 const char *sp = strrchr(v[0], '.'); 475 const char *extension = ""; 476 if (!sp || stricmp(sp, ".exe")) 477 extension = ".exe"; /* Add an exe so CreateProcess will work */ 478 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 479 commandname[MAX_PATH]='\0'; 480 #endif 481 for (i=1; i<c; ++i) { 482 if (v[i][0] == '-') { 483 if (!strcmp(v[i], "--RUNNING-FORKED")) { 484 opt_forked = 1; 485 } else if (!strcmp(v[i], "--no-fork")) { 486 opt_nofork = 1; 487 } else if (!strcmp(v[i], "--quiet")) { 488 opt_verbosity = -1; 489 verbosity_flag = "--quiet"; 490 } else if (!strcmp(v[i], "--verbose")) { 491 opt_verbosity = 2; 492 verbosity_flag = "--verbose"; 493 } else if (!strcmp(v[i], "--terse")) { 494 opt_verbosity = 0; 495 verbosity_flag = "--terse"; 496 } else if (!strcmp(v[i], "--help")) { 497 usage(groups, 0); 498 } else if (!strcmp(v[i], "--list-tests")) { 499 usage(groups, 1); 500 } else if (!strcmp(v[i], "--timeout")) { 501 ++i; 502 if (i >= c) { 503 fprintf(stderr, "--timeout requires argument\n"); 504 return -1; 505 } 506 opt_timeout = (unsigned)atoi(v[i]); 507 } else { 508 fprintf(stderr, "Unknown option %s. Try --help\n", v[i]); 509 return -1; 510 } 511 } else { 512 int r = process_test_option(groups, v[i]); 513 if (r<0) 514 return -1; 515 n += r; 516 } 517 } 518 if (!n) 519 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 520 521 #ifdef _IONBF 522 setvbuf(stdout, NULL, _IONBF, 0); 523 #endif 524 525 ++in_tinytest_main; 526 for (i = 0; groups[i].prefix; ++i) { 527 struct testgroup_t *group = &groups[i]; 528 for (j = 0; group->cases[j].name; ++j) { 529 struct testcase_t *testcase = &group->cases[j]; 530 int test_attempts = 3; 531 int test_ret_err; 532 533 if (!(testcase->flags & TT_ENABLED_)) 534 continue; 535 536 for (;;) { 537 test_ret_err = testcase_run_one(group, testcase); 538 539 if (test_ret_err == OK) 540 break; 541 if (!(testcase->flags & TT_RETRIABLE)) 542 break; 543 printf("\n [RETRYING %s (%i)]\n", testcase->name, test_attempts); 544 if (!test_attempts--) 545 break; 546 } 547 548 switch (test_ret_err) { 549 case OK: ++n_ok; break; 550 case SKIP: ++n_skipped; break; 551 default: ++n_bad; break; 552 } 553 } 554 } 555 556 --in_tinytest_main; 557 558 if (opt_verbosity==0) 559 puts(""); 560 561 if (n_bad) 562 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 563 n_bad+n_ok,n_skipped); 564 else if (opt_verbosity >= 1) 565 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 566 567 return (n_bad == 0) ? 0 : 1; 568 } 569 570 int 571 tinytest_get_verbosity_(void) 572 { 573 return opt_verbosity; 574 } 575 576 void 577 tinytest_set_test_failed_(void) 578 { 579 if (opt_verbosity <= 0 && cur_test_name) { 580 if (opt_verbosity==0) puts(""); 581 printf("%s%s: ", cur_test_prefix, cur_test_name); 582 cur_test_name = NULL; 583 } 584 cur_test_outcome = FAIL; 585 } 586 587 void 588 tinytest_set_test_skipped_(void) 589 { 590 if (cur_test_outcome==OK) 591 cur_test_outcome = SKIP; 592 } 593 594 char * 595 tinytest_format_hex_(const void *val_, unsigned long len) 596 { 597 const unsigned char *val = val_; 598 char *result, *cp; 599 size_t i; 600 601 if (!val) 602 return strdup("null"); 603 if (!(result = malloc(len*2+1))) 604 return strdup("<allocation failure>"); 605 cp = result; 606 for (i=0;i<len;++i) { 607 *cp++ = "0123456789ABCDEF"[val[i] >> 4]; 608 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; 609 } 610 *cp = 0; 611 return result; 612 } 613