1 /* $NetBSD: tinytest.c,v 1.6 2020/05/25 20:47:34 christos Exp $ */ 2 3 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 #ifdef TINYTEST_LOCAL 28 #include "tinytest_local.h" 29 #endif 30 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <assert.h> 35 36 #ifndef NO_FORKING 37 38 #ifdef _WIN32 39 #include <windows.h> 40 #else 41 #include <sys/types.h> 42 #include <sys/wait.h> 43 #include <unistd.h> 44 #endif 45 46 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 47 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 48 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 49 /* Workaround for a stupid bug in OSX 10.6 */ 50 #define FORK_BREAKS_GCOV 51 #include <vproc.h> 52 #endif 53 #endif 54 55 #endif /* !NO_FORKING */ 56 57 #ifndef __GNUC__ 58 #define __attribute__(x) 59 #endif 60 61 #include "tinytest.h" 62 #include "tinytest_macros.h" 63 64 #define LONGEST_TEST_NAME 16384 65 66 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 67 static int n_ok = 0; /**< Number of tests that have passed */ 68 static int n_bad = 0; /**< Number of tests that have failed. */ 69 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 70 71 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 72 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 73 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 74 const char *verbosity_flag = ""; 75 76 const struct testlist_alias_t *cfg_aliases=NULL; 77 78 enum outcome { SKIP=2, OK=1, FAIL=0 }; 79 static enum outcome cur_test_outcome = 0; 80 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 81 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 82 const char *cur_test_name = NULL; 83 84 #ifdef _WIN32 85 /* Copy of argv[0] for win32. */ 86 static char commandname[MAX_PATH+1]; 87 #endif 88 89 static void usage(struct testgroup_t *groups, int list_groups) 90 __attribute__((noreturn)); 91 static int process_test_option(struct testgroup_t *groups, const char *test); 92 93 static enum outcome 94 testcase_run_bare_(const struct testcase_t *testcase) 95 { 96 void *env = NULL; 97 int outcome; 98 if (testcase->setup) { 99 env = testcase->setup->setup_fn(testcase); 100 if (!env) 101 return FAIL; 102 else if (env == (void*)TT_SKIP) 103 return SKIP; 104 } 105 106 cur_test_outcome = OK; 107 testcase->fn(env); 108 outcome = cur_test_outcome; 109 110 if (testcase->setup) { 111 if (testcase->setup->cleanup_fn(testcase, env) == 0) 112 outcome = FAIL; 113 } 114 115 return outcome; 116 } 117 118 #define MAGIC_EXITCODE 42 119 120 #ifndef NO_FORKING 121 122 static enum outcome 123 testcase_run_forked_(const struct testgroup_t *group, 124 const struct testcase_t *testcase) 125 { 126 #ifdef _WIN32 127 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 128 we'll invoke our own exe (whose name we recall from the command 129 line) with a command line that tells it to run just the test we 130 want, and this time without forking. 131 132 (No, threads aren't an option. The whole point of forking is to 133 share no state between tests.) 134 */ 135 int ok; 136 char buffer[LONGEST_TEST_NAME+256]; 137 STARTUPINFOA si; 138 PROCESS_INFORMATION info; 139 DWORD exitcode; 140 141 if (!in_tinytest_main) { 142 printf("\nERROR. On Windows, testcase_run_forked_ must be" 143 " called from within tinytest_main.\n"); 144 abort(); 145 } 146 if (opt_verbosity>0) 147 printf("[forking] "); 148 149 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 150 commandname, verbosity_flag, group->prefix, testcase->name); 151 152 memset(&si, 0, sizeof(si)); 153 memset(&info, 0, sizeof(info)); 154 si.cb = sizeof(si); 155 156 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 157 0, NULL, NULL, &si, &info); 158 if (!ok) { 159 printf("CreateProcess failed!\n"); 160 return 0; 161 } 162 WaitForSingleObject(info.hProcess, INFINITE); 163 GetExitCodeProcess(info.hProcess, &exitcode); 164 CloseHandle(info.hProcess); 165 CloseHandle(info.hThread); 166 if (exitcode == 0) 167 return OK; 168 else if (exitcode == MAGIC_EXITCODE) 169 return SKIP; 170 else 171 return FAIL; 172 #else 173 int outcome_pipe[2]; 174 pid_t pid; 175 (void)group; 176 177 if (pipe(outcome_pipe)) 178 perror("opening pipe"); 179 180 if (opt_verbosity>0) 181 printf("[forking] "); 182 pid = fork(); 183 #ifdef FORK_BREAKS_GCOV 184 vproc_transaction_begin(0); 185 #endif 186 if (!pid) { 187 /* child. */ 188 int test_r, write_r; 189 char b[1]; 190 close(outcome_pipe[0]); 191 test_r = testcase_run_bare_(testcase); 192 assert(0<=(int)test_r && (int)test_r<=2); 193 b[0] = "NYS"[test_r]; 194 write_r = (int)write(outcome_pipe[1], b, 1); 195 if (write_r != 1) { 196 perror("write outcome to pipe"); 197 exit(1); 198 } 199 exit(0); 200 return FAIL; /* unreachable */ 201 } else { 202 /* parent */ 203 int status, r; 204 char b[1]; 205 /* Close this now, so that if the other side closes it, 206 * our read fails. */ 207 close(outcome_pipe[1]); 208 r = (int)read(outcome_pipe[0], b, 1); 209 if (r == 0) { 210 printf("[Lost connection!] "); 211 return 0; 212 } else if (r != 1) { 213 perror("read outcome from pipe"); 214 } 215 waitpid(pid, &status, 0); 216 close(outcome_pipe[0]); 217 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 218 } 219 #endif 220 } 221 222 #endif /* !NO_FORKING */ 223 224 int 225 testcase_run_one(const struct testgroup_t *group, 226 const struct testcase_t *testcase) 227 { 228 enum outcome outcome; 229 230 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 231 if (opt_verbosity>0) 232 printf("%s%s: %s\n", 233 group->prefix, testcase->name, 234 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 235 ++n_skipped; 236 return SKIP; 237 } 238 239 if (opt_verbosity>0 && !opt_forked) { 240 printf("%s%s: ", group->prefix, testcase->name); 241 } else { 242 if (opt_verbosity==0) printf("."); 243 cur_test_prefix = group->prefix; 244 cur_test_name = testcase->name; 245 } 246 247 #ifndef NO_FORKING 248 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 249 outcome = testcase_run_forked_(group, testcase); 250 } else { 251 #else 252 { 253 #endif 254 outcome = testcase_run_bare_(testcase); 255 } 256 257 if (outcome == OK) { 258 ++n_ok; 259 if (opt_verbosity>0 && !opt_forked) 260 puts(opt_verbosity==1?"OK":""); 261 } else if (outcome == SKIP) { 262 ++n_skipped; 263 if (opt_verbosity>0 && !opt_forked) 264 puts("SKIPPED"); 265 } else { 266 ++n_bad; 267 if (!opt_forked) 268 printf("\n [%s FAILED]\n", testcase->name); 269 } 270 271 if (opt_forked) { 272 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 273 return 1; /* unreachable */ 274 } else { 275 return (int)outcome; 276 } 277 } 278 279 int 280 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 281 { 282 int i, j; 283 size_t length = LONGEST_TEST_NAME; 284 char fullname[LONGEST_TEST_NAME]; 285 int found=0; 286 if (strstr(arg, "..")) 287 length = strstr(arg,"..")-arg; 288 for (i=0; groups[i].prefix; ++i) { 289 for (j=0; groups[i].cases[j].name; ++j) { 290 struct testcase_t *testcase = &groups[i].cases[j]; 291 snprintf(fullname, sizeof(fullname), "%s%s", 292 groups[i].prefix, testcase->name); 293 if (!flag) { /* Hack! */ 294 printf(" %s", fullname); 295 if (testcase->flags & TT_OFF_BY_DEFAULT) 296 puts(" (Off by default)"); 297 else if (testcase->flags & TT_SKIP) 298 puts(" (DISABLED)"); 299 else 300 puts(""); 301 } 302 if (!strncmp(fullname, arg, length)) { 303 if (set) 304 testcase->flags |= flag; 305 else 306 testcase->flags &= ~flag; 307 ++found; 308 } 309 } 310 } 311 return found; 312 } 313 314 static void 315 usage(struct testgroup_t *groups, int list_groups) 316 { 317 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 318 puts(" Specify tests by name, or using a prefix ending with '..'"); 319 puts(" To skip a test, prefix its name with a colon."); 320 puts(" To enable a disabled test, prefix its name with a plus."); 321 puts(" Use --list-tests for a list of tests."); 322 if (list_groups) { 323 puts("Known tests are:"); 324 tinytest_set_flag_(groups, "..", 1, 0); 325 } 326 exit(0); 327 } 328 329 static int 330 process_test_alias(struct testgroup_t *groups, const char *test) 331 { 332 int i, j, n, r; 333 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 334 if (!strcmp(cfg_aliases[i].name, test)) { 335 n = 0; 336 for (j = 0; cfg_aliases[i].tests[j]; ++j) { 337 r = process_test_option(groups, cfg_aliases[i].tests[j]); 338 if (r<0) 339 return -1; 340 n += r; 341 } 342 return n; 343 } 344 } 345 printf("No such test alias as @%s!",test); 346 return -1; 347 } 348 349 static int 350 process_test_option(struct testgroup_t *groups, const char *test) 351 { 352 int flag = TT_ENABLED_; 353 int n = 0; 354 if (test[0] == '@') { 355 return process_test_alias(groups, test + 1); 356 } else if (test[0] == ':') { 357 ++test; 358 flag = TT_SKIP; 359 } else if (test[0] == '+') { 360 ++test; 361 ++n; 362 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 363 printf("No such test as %s!\n", test); 364 return -1; 365 } 366 } else { 367 ++n; 368 } 369 if (!tinytest_set_flag_(groups, test, 1, flag)) { 370 printf("No such test as %s!\n", test); 371 return -1; 372 } 373 return n; 374 } 375 376 void 377 tinytest_set_aliases(const struct testlist_alias_t *aliases) 378 { 379 cfg_aliases = aliases; 380 } 381 382 int 383 tinytest_main(int c, const char **v, struct testgroup_t *groups) 384 { 385 int i, j, n=0; 386 387 #ifdef _WIN32 388 const char *sp = strrchr(v[0], '.'); 389 const char *extension = ""; 390 if (!sp || stricmp(sp, ".exe")) 391 extension = ".exe"; /* Add an exe so CreateProcess will work */ 392 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 393 commandname[MAX_PATH]='\0'; 394 #endif 395 for (i=1; i<c; ++i) { 396 if (v[i][0] == '-') { 397 if (!strcmp(v[i], "--RUNNING-FORKED")) { 398 opt_forked = 1; 399 } else if (!strcmp(v[i], "--no-fork")) { 400 opt_nofork = 1; 401 } else if (!strcmp(v[i], "--quiet")) { 402 opt_verbosity = -1; 403 verbosity_flag = "--quiet"; 404 } else if (!strcmp(v[i], "--verbose")) { 405 opt_verbosity = 2; 406 verbosity_flag = "--verbose"; 407 } else if (!strcmp(v[i], "--terse")) { 408 opt_verbosity = 0; 409 verbosity_flag = "--terse"; 410 } else if (!strcmp(v[i], "--help")) { 411 usage(groups, 0); 412 } else if (!strcmp(v[i], "--list-tests")) { 413 usage(groups, 1); 414 } else { 415 printf("Unknown option %s. Try --help\n",v[i]); 416 return -1; 417 } 418 } else { 419 int r = process_test_option(groups, v[i]); 420 if (r<0) 421 return -1; 422 n += r; 423 } 424 } 425 if (!n) 426 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 427 428 #ifdef _IONBF 429 setvbuf(stdout, NULL, _IONBF, 0); 430 #endif 431 432 ++in_tinytest_main; 433 for (i=0; groups[i].prefix; ++i) 434 for (j=0; groups[i].cases[j].name; ++j) 435 if (groups[i].cases[j].flags & TT_ENABLED_) 436 testcase_run_one(&groups[i], 437 &groups[i].cases[j]); 438 439 --in_tinytest_main; 440 441 if (opt_verbosity==0) 442 puts(""); 443 444 if (n_bad) 445 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 446 n_bad+n_ok,n_skipped); 447 else if (opt_verbosity >= 1) 448 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 449 450 return (n_bad == 0) ? 0 : 1; 451 } 452 453 int 454 tinytest_get_verbosity_(void) 455 { 456 return opt_verbosity; 457 } 458 459 void 460 tinytest_set_test_failed_(void) 461 { 462 if (opt_verbosity <= 0 && cur_test_name) { 463 if (opt_verbosity==0) puts(""); 464 printf("%s%s: ", cur_test_prefix, cur_test_name); 465 cur_test_name = NULL; 466 } 467 cur_test_outcome = 0; 468 } 469 470 void 471 tinytest_set_test_skipped_(void) 472 { 473 if (cur_test_outcome==OK) 474 cur_test_outcome = SKIP; 475 } 476 477 char * 478 tinytest_format_hex_(const void *val_, unsigned long len) 479 { 480 const unsigned char *val = val_; 481 char *result, *cp; 482 size_t i; 483 484 if (!val) 485 return strdup("null"); 486 if (!(result = malloc(len*2+1))) 487 return strdup("<allocation failure>"); 488 cp = result; 489 for (i=0;i<len;++i) { 490 *cp++ = "0123456789ABCDEF"[val[i] >> 4]; 491 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; 492 } 493 *cp = 0; 494 return result; 495 } 496