1// 2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 3// See https://llvm.org/LICENSE.txt for license information. 4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 5 6// 7// testfilerunner.m 8// testObjects 9// 10// Created by Blaine Garst on 9/24/08. 11// 12 13#import "testfilerunner.h" 14#import <Foundation/Foundation.h> 15#include <stdio.h> 16#include <unistd.h> 17#include <fcntl.h> 18#include <string.h> 19#include <stdlib.h> 20#include <stdbool.h> 21 22bool Everything = false; // do it also with 3 levels of optimization 23bool DoClang = false; 24 25static bool isDirectory(char *path); 26static bool isExecutable(char *path); 27static bool isYounger(char *source, char *binary); 28static bool readErrorFile(char *buffer, const char *from); 29 30__strong char *gcstrcpy2(__strong const char *arg, char *endp) { 31 unsigned size = endp - arg + 1; 32 __strong char *result = NSAllocateCollectable(size, 0); 33 strncpy(result, arg, size); 34 result[size-1] = 0; 35 return result; 36} 37__strong char *gcstrcpy1(__strong char *arg) { 38 unsigned size = strlen(arg) + 1; 39 __strong char *result = NSAllocateCollectable(size, 0); 40 strncpy(result, arg, size); 41 result[size-1] = 0; 42 return result; 43} 44 45@implementation TestFileExe 46 47@synthesize options, compileLine, shouldFail, binaryName, sourceName; 48@synthesize generator; 49@synthesize libraryPath, frameworkPath; 50 51- (NSString *)description { 52 NSMutableString *result = [NSMutableString new]; 53 if (shouldFail) [result appendString:@"fail"]; 54 for (id x in compileLine) { 55 [result appendString:[NSString stringWithFormat:@" %s", (char *)x]]; 56 } 57 return result; 58} 59 60- (__strong char *)radar { 61 return generator.radar; 62} 63 64- (bool) compileUnlessExists:(bool)skip { 65 if (shouldFail) { 66 printf("don't use this to compile anymore!\n"); 67 return false; 68 } 69 if (skip && isExecutable(binaryName) && !isYounger(sourceName, binaryName)) return true; 70 int argc = [compileLine count]; 71 char *argv[argc+1]; 72 for (int i = 0; i < argc; ++i) 73 argv[i] = (char *)[compileLine pointerAtIndex:i]; 74 argv[argc] = NULL; 75 pid_t child = fork(); 76 if (child == 0) { 77 execv(argv[0], argv); 78 exit(10); // shouldn't happen 79 } 80 if (child < 0) { 81 printf("fork failed\n"); 82 return false; 83 } 84 int status = 0; 85 pid_t deadchild = wait(&status); 86 if (deadchild != child) { 87 printf("wait got %d instead of %d\n", deadchild, child); 88 exit(1); 89 } 90 if (WEXITSTATUS(status) == 0) { 91 return true; 92 } 93 printf("run failed\n"); 94 return false; 95} 96 97bool lookforIn(char *lookfor, const char *format, pid_t child) { 98 char buffer[512]; 99 char got[512]; 100 sprintf(buffer, format, child); 101 bool gotOutput = readErrorFile(got, buffer); 102 if (!gotOutput) { 103 printf("**** didn't get an output file %s to analyze!!??\n", buffer); 104 return false; 105 } 106 char *where = strstr(got, lookfor); 107 if (!where) { 108 printf("didn't find '%s' in output file %s\n", lookfor, buffer); 109 return false; 110 } 111 unlink(buffer); 112 return true; 113} 114 115- (bool) compileWithExpectedFailure { 116 if (!shouldFail) { 117 printf("Why am I being called?\n"); 118 return false; 119 } 120 int argc = [compileLine count]; 121 char *argv[argc+1]; 122 for (int i = 0; i < argc; ++i) 123 argv[i] = (char *)[compileLine pointerAtIndex:i]; 124 argv[argc] = NULL; 125 pid_t child = fork(); 126 char buffer[512]; 127 if (child == 0) { 128 // in child 129 sprintf(buffer, "/tmp/errorfile_%d", getpid()); 130 close(1); 131 int fd = creat(buffer, 0777); 132 if (fd != 1) { 133 fprintf(stderr, "didn't open custom error file %s as 1, got %d\n", buffer, fd); 134 exit(1); 135 } 136 close(2); 137 dup(1); 138 int result = execv(argv[0], argv); 139 exit(10); 140 } 141 if (child < 0) { 142 printf("fork failed\n"); 143 return false; 144 } 145 int status = 0; 146 pid_t deadchild = wait(&status); 147 if (deadchild != child) { 148 printf("wait got %d instead of %d\n", deadchild, child); 149 exit(11); 150 } 151 if (WIFEXITED(status)) { 152 if (WEXITSTATUS(status) == 0) { 153 return false; 154 } 155 } 156 else { 157 printf("***** compiler borked/ICEd/died unexpectedly (status %x)\n", status); 158 return false; 159 } 160 char *error = generator.errorString; 161 162 if (!error) return true; 163#if 0 164 char got[512]; 165 sprintf(buffer, "/tmp/errorfile_%d", child); 166 bool gotOutput = readErrorFile(got, buffer); 167 if (!gotOutput) { 168 printf("**** didn't get an error file %s to analyze!!??\n", buffer); 169 return false; 170 } 171 char *where = strstr(got, error); 172 if (!where) { 173 printf("didn't find '%s' in error file %s\n", error, buffer); 174 return false; 175 } 176 unlink(buffer); 177#else 178 if (!lookforIn(error, "/tmp/errorfile_%d", child)) return false; 179#endif 180 return true; 181} 182 183- (bool) run { 184 if (shouldFail) return true; 185 if (sizeof(long) == 4 && options & Do64) { 186 return true; // skip 64-bit tests 187 } 188 int argc = 1; 189 char *argv[argc+1]; 190 argv[0] = binaryName; 191 argv[argc] = NULL; 192 pid_t child = fork(); 193 if (child == 0) { 194 // set up environment 195 char lpath[1024]; 196 char fpath[1024]; 197 char *myenv[3]; 198 int counter = 0; 199 if (libraryPath) { 200 sprintf(lpath, "DYLD_LIBRARY_PATH=%s", libraryPath); 201 myenv[counter++] = lpath; 202 } 203 if (frameworkPath) { 204 sprintf(fpath, "DYLD_FRAMEWORK_PATH=%s", frameworkPath); 205 myenv[counter++] = fpath; 206 } 207 myenv[counter] = NULL; 208 if (generator.warningString) { 209 // set up stdout/stderr 210 char outfile[1024]; 211 sprintf(outfile, "/tmp/stdout_%d", getpid()); 212 close(2); 213 close(1); 214 creat(outfile, 0700); 215 dup(1); 216 } 217 execve(argv[0], argv, myenv); 218 exit(10); // shouldn't happen 219 } 220 if (child < 0) { 221 printf("fork failed\n"); 222 return false; 223 } 224 int status = 0; 225 pid_t deadchild = wait(&status); 226 if (deadchild != child) { 227 printf("wait got %d instead of %d\n", deadchild, child); 228 exit(1); 229 } 230 if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { 231 if (generator.warningString) { 232 if (!lookforIn(generator.warningString, "/tmp/stdout_%d", child)) return false; 233 } 234 return true; 235 } 236 printf("**** run failed for %s\n", binaryName); 237 return false; 238} 239 240@end 241 242@implementation TestFileExeGenerator 243@synthesize filename, compilerPath, errorString; 244@synthesize hasObjC, hasRR, hasGC, hasCPlusPlus, wantsC99, supposedToNotCompile, open, wants32, wants64; 245@synthesize radar; 246@synthesize warningString; 247 248- (void)setFilename:(__strong char *)name { 249 filename = gcstrcpy1(name); 250} 251- (void)setCompilerPath:(__strong char *)name { 252 compilerPath = gcstrcpy1(name); 253} 254 255- (void)forMostThings:(NSMutableArray *)lines options:(int)options { 256 TestFileExe *item = nil; 257 item = [self lineForOptions:options]; 258 if (item) [lines addObject:item]; 259 item = [self lineForOptions:options|Do64]; 260 if (item) [lines addObject:item]; 261 item = [self lineForOptions:options|DoCPP]; 262 if (item) [lines addObject:item]; 263 item = [self lineForOptions:options|Do64|DoCPP]; 264 if (item) [lines addObject:item]; 265} 266 267/* 268 DoDashG = (1 << 8), 269 DoDashO = (1 << 9), 270 DoDashOs = (1 << 10), 271 DoDashO2 = (1 << 11), 272*/ 273 274- (void)forAllThings:(NSMutableArray *)lines options:(int)options { 275 [self forMostThings:lines options:options]; 276 if (!Everything) { 277 return; 278 } 279 // now do it with three explicit optimization flags 280 [self forMostThings:lines options:options | DoDashO]; 281 [self forMostThings:lines options:options | DoDashOs]; 282 [self forMostThings:lines options:options | DoDashO2]; 283} 284 285- (NSArray *)allLines { 286 NSMutableArray *result = [NSMutableArray new]; 287 TestFileExe *item = nil; 288 289 int options = 0; 290 [self forAllThings:result options:0]; 291 [self forAllThings:result options:DoOBJC | DoRR]; 292 [self forAllThings:result options:DoOBJC | DoGC]; 293 [self forAllThings:result options:DoOBJC | DoGCRR]; 294 //[self forAllThings:result options:DoOBJC | DoRRGC]; 295 296 return result; 297} 298 299- (void)addLibrary:(const char *)dashLSomething { 300 if (!extraLibraries) { 301 extraLibraries = [NSPointerArray pointerArrayWithOptions: 302 NSPointerFunctionsStrongMemory | 303 NSPointerFunctionsCStringPersonality]; 304 } 305 [extraLibraries addPointer:(void *)dashLSomething]; 306} 307 308- (TestFileExe *)lineForOptions:(int)options { // nil if no can do 309 if (hasObjC && !(options & DoOBJC)) return nil; 310 if (hasCPlusPlus && !(options & DoCPP)) return nil; 311 if (hasObjC) { 312 if (!hasGC && (options & (DoGC|DoGCRR))) return nil; // not smart enough 313 if (!hasRR && (options & (DoRR|DoRRGC))) return nil; 314 } 315 NSPointerArray *pa = [NSPointerArray pointerArrayWithOptions: 316 NSPointerFunctionsStrongMemory | 317 NSPointerFunctionsCStringPersonality]; 318 // construct path 319 char path[512]; 320 path[0] = 0; 321 if (!compilerPath) compilerPath = "/usr/bin"; 322 if (compilerPath) { 323 strcat(path, compilerPath); 324 strcat(path, "/"); 325 } 326 if (options & DoCPP) { 327 strcat(path, DoClang ? "clang++" : "g++-4.2"); 328 } 329 else { 330 strcat(path, DoClang ? "clang" : "gcc-4.2"); 331 } 332 [pa addPointer:gcstrcpy1(path)]; 333 if (options & DoOBJC) { 334 if (options & DoCPP) { 335 [pa addPointer:"-ObjC++"]; 336 } 337 else { 338 [pa addPointer:"-ObjC"]; 339 } 340 } 341 [pa addPointer:"-g"]; 342 if (options & DoDashO) [pa addPointer:"-O"]; 343 else if (options & DoDashO2) [pa addPointer:"-O2"]; 344 else if (options & DoDashOs) [pa addPointer:"-Os"]; 345 if (wantsC99 && (! (options & DoCPP))) { 346 [pa addPointer:"-std=c99"]; 347 [pa addPointer:"-fblocks"]; 348 } 349 [pa addPointer:"-arch"]; 350 [pa addPointer: (options & Do64) ? "x86_64" : "i386"]; 351 352 if (options & DoOBJC) { 353 switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) { 354 case DoRR: 355 break; 356 case DoGC: 357 [pa addPointer:"-fobjc-gc-only"]; 358 break; 359 case DoGCRR: 360 [pa addPointer:"-fobjc-gc"]; 361 break; 362 case DoRRGC: 363 printf("DoRRGC unsupported right now\n"); 364 [pa addPointer:"-c"]; 365 return nil; 366 } 367 [pa addPointer:"-framework"]; 368 [pa addPointer:"Foundation"]; 369 } 370 [pa addPointer:gcstrcpy1(filename)]; 371 [pa addPointer:"-o"]; 372 373 path[0] = 0; 374 strcat(path, filename); 375 strcat(path, "."); 376 strcat(path, (options & Do64) ? "64" : "32"); 377 if (options & DoOBJC) { 378 switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) { 379 case DoRR: strcat(path, "-rr"); break; 380 case DoGC: strcat(path, "-gconly"); break; 381 case DoGCRR: strcat(path, "-gcrr"); break; 382 case DoRRGC: strcat(path, "-rrgc"); break; 383 } 384 } 385 if (options & DoCPP) strcat(path, "++"); 386 if (options & DoDashO) strcat(path, "-O"); 387 else if (options & DoDashO2) strcat(path, "-O2"); 388 else if (options & DoDashOs) strcat(path, "-Os"); 389 if (wantsC99) strcat(path, "-C99"); 390 strcat(path, DoClang ? "-clang" : "-gcc"); 391 strcat(path, "-bin"); 392 TestFileExe *result = [TestFileExe new]; 393 result.binaryName = gcstrcpy1(path); // could snarf copy in pa 394 [pa addPointer:result.binaryName]; 395 for (id cString in extraLibraries) { 396 [pa addPointer:cString]; 397 } 398 399 result.sourceName = gcstrcpy1(filename); // could snarf copy in pa 400 result.compileLine = pa; 401 result.options = options; 402 result.shouldFail = supposedToNotCompile; 403 result.generator = self; 404 return result; 405} 406 407+ (NSArray *)generatorsFromPath:(NSString *)path { 408 FILE *fp = fopen([path fileSystemRepresentation], "r"); 409 if (fp == NULL) return nil; 410 NSArray *result = [self generatorsFromFILE:fp]; 411 fclose(fp); 412 return result; 413} 414 415#define LOOKFOR "CON" "FIG" 416 417char *__strong parseRadar(char *line) { 418 line = strstr(line, "rdar:"); // returns beginning 419 char *endp = line + strlen("rdar:"); 420 while (*endp && *endp != ' ' && *endp != '\n') 421 ++endp; 422 return gcstrcpy2(line, endp); 423} 424 425- (void)parseLibraries:(const char *)line { 426 start: 427 line = strstr(line, "-l"); 428 char *endp = (char *)line + 2; 429 while (*endp && *endp != ' ' && *endp != '\n') 430 ++endp; 431 [self addLibrary:gcstrcpy2(line, endp)]; 432 if (strstr(endp, "-l")) { 433 line = endp; 434 goto start; 435 } 436} 437 438+ (TestFileExeGenerator *)generatorFromLine:(char *)line filename:(char *)filename { 439 TestFileExeGenerator *item = [TestFileExeGenerator new]; 440 item.filename = gcstrcpy1(filename); 441 if (strstr(line, "GC")) item.hasGC = true; 442 if (strstr(line, "RR")) item.hasRR = true; 443 if (strstr(line, "C++")) item.hasCPlusPlus = true; 444 if (strstr(line, "-C99")) { 445 item.wantsC99 = true; 446 } 447 if (strstr(line, "64")) item.wants64 = true; 448 if (strstr(line, "32")) item.wants32 = true; 449 if (strstr(line, "-l")) [item parseLibraries:line]; 450 if (strstr(line, "open")) item.open = true; 451 if (strstr(line, "FAIL")) item.supposedToNotCompile = true; // old 452 // compile time error 453 if (strstr(line, "error:")) { 454 item.supposedToNotCompile = true; 455 // zap newline 456 char *error = strstr(line, "error:") + strlen("error:"); 457 // make sure we have something before the newline 458 char *newline = strstr(error, "\n"); 459 if (newline && ((newline-error) > 1)) { 460 *newline = 0; 461 item.errorString = gcstrcpy1(strstr(line, "error:") + strlen("error: ")); 462 } 463 } 464 // run time warning 465 if (strstr(line, "runtime:")) { 466 // zap newline 467 char *error = strstr(line, "runtime:") + strlen("runtime:"); 468 // make sure we have something before the newline 469 char *newline = strstr(error, "\n"); 470 if (newline && ((newline-error) > 1)) { 471 *newline = 0; 472 item.warningString = gcstrcpy1(strstr(line, "runtime:") + strlen("runtime:")); 473 } 474 } 475 if (strstr(line, "rdar:")) item.radar = parseRadar(line); 476 if (item.hasGC || item.hasRR) item.hasObjC = true; 477 if (!item.wants32 && !item.wants64) { // give them both if they ask for neither 478 item.wants32 = item.wants64 = true; 479 } 480 return item; 481} 482 483+ (NSArray *)generatorsFromFILE:(FILE *)fp { 484 NSMutableArray *result = [NSMutableArray new]; 485 // pretend this is a grep LOOKFOR *.[cmCM][cmCM] input 486 // look for 487 // filename: ... LOOKFOR [GC] [RR] [C++] [FAIL ...] 488 char buf[512]; 489 while (fgets(buf, 512, fp)) { 490 char *config = strstr(buf, LOOKFOR); 491 if (!config) continue; 492 char *filename = buf; 493 char *end = strchr(buf, ':'); 494 *end = 0; 495 [result addObject:[self generatorFromLine:config filename:filename]]; 496 } 497 return result; 498} 499 500+ (TestFileExeGenerator *)generatorFromFilename:(char *)filename { 501 FILE *fp = fopen(filename, "r"); 502 if (!fp) { 503 printf("didn't open %s!!\n", filename); 504 return nil; 505 } 506 char buf[512]; 507 while (fgets(buf, 512, fp)) { 508 char *config = strstr(buf, LOOKFOR); 509 if (!config) continue; 510 fclose(fp); 511 return [self generatorFromLine:config filename:filename]; 512 } 513 fclose(fp); 514 // guess from filename 515 char *ext = strrchr(filename, '.'); 516 if (!ext) return nil; 517 TestFileExeGenerator *result = [TestFileExeGenerator new]; 518 result.filename = gcstrcpy1(filename); 519 if (!strncmp(ext, ".m", 2)) { 520 result.hasObjC = true; 521 result.hasRR = true; 522 result.hasGC = true; 523 } 524 else if (!strcmp(ext, ".c")) { 525 ; 526 } 527 else if (!strcmp(ext, ".M") || !strcmp(ext, ".mm")) { 528 result.hasObjC = true; 529 result.hasRR = true; 530 result.hasGC = true; 531 result.hasCPlusPlus = true; 532 } 533 else if (!strcmp(ext, ".cc") 534 || !strcmp(ext, ".cp") 535 || !strcmp(ext, ".cxx") 536 || !strcmp(ext, ".cpp") 537 || !strcmp(ext, ".CPP") 538 || !strcmp(ext, ".c++") 539 || !strcmp(ext, ".C")) { 540 result.hasCPlusPlus = true; 541 } 542 else { 543 printf("unknown extension, file %s ignored\n", filename); 544 result = nil; 545 } 546 return result; 547 548} 549 550- (NSString *)description { 551 return [NSString stringWithFormat:@"%s: %s%s%s%s%s%s", 552 filename, 553 LOOKFOR, 554 hasGC ? " GC" : "", 555 hasRR ? " RR" : "", 556 hasCPlusPlus ? " C++" : "", 557 wantsC99 ? "C99" : "", 558 supposedToNotCompile ? " FAIL" : ""]; 559} 560 561@end 562 563void printDetails(NSArray *failures, const char *whatAreThey) { 564 if ([failures count]) { 565 NSMutableString *output = [NSMutableString new]; 566 printf("%s:\n", whatAreThey); 567 for (TestFileExe *line in failures) { 568 printf("%s", line.binaryName); 569 char *radar = line.generator.radar; 570 if (radar) 571 printf(" (due to %s?),", radar); 572 printf(" recompile via:\n%s\n\n", line.description.UTF8String); 573 } 574 printf("\n"); 575 } 576} 577 578void help(const char *whoami) { 579 printf("Usage: %s [-fast] [-e] [-dyld librarypath] [gcc4.2dir] [-- | source1 ...]\n", whoami); 580 printf(" -fast don't recompile if binary younger than source\n"); 581 printf(" -open only run tests that are thought to still be unresolved\n"); 582 printf(" -clang use the clang and clang++ compilers\n"); 583 printf(" -e compile all variations also with -Os, -O2, -O3\n"); 584 printf(" -dyld p override DYLD_LIBRARY_PATH and DYLD_FRAMEWORK_PATH to p when running tests\n"); 585 printf(" <compilerpath> directory containing gcc-4.2 (or clang) that you wish to use to compile the tests\n"); 586 printf(" -- assume stdin is a grep CON" "FIG across the test sources\n"); 587 printf(" otherwise treat each remaining argument as a single test file source\n"); 588 printf("%s will compile and run individual test files under a variety of compilers, c, obj-c, c++, and objc++\n", whoami); 589 printf(" .c files are compiled with all four compilers\n"); 590 printf(" .m files are compiled with objc and objc++ compilers\n"); 591 printf(" .C files are compiled with c++ and objc++ compilers\n"); 592 printf(" .M files are compiled only with the objc++ compiler\n"); 593 printf("(actually all forms of extensions recognized by the compilers are honored, .cc, .c++ etc.)\n"); 594 printf("\nTest files should run to completion with no output and exit (return) 0 on success.\n"); 595 printf("Further they should be able to be compiled and run with GC on or off and by the C++ compilers\n"); 596 printf("A line containing the string CON" "FIG within the source enables restrictions to the above assumptions\n"); 597 printf("and other options.\n"); 598 printf("Following CON" "FIG the string\n"); 599 printf(" C++ restricts the test to only be run by c++ and objc++ compilers\n"); 600 printf(" GC restricts the test to only be compiled and run with GC on\n"); 601 printf(" RR (retain/release) restricts the test to only be compiled and run with GC off\n"); 602 printf("Additionally,\n"); 603 printf(" -C99 restricts the C versions of the test to -fstd=c99 -fblocks\n"); 604 printf(" -O adds the -O optimization level\n"); 605 printf(" -O2 adds the -O2 optimization level\n"); 606 printf(" -Os adds the -Os optimization level\n"); 607 printf("Files that are known to exhibit unresolved problems can provide the term \"open\" and this can"); 608 printf("in turn allow highlighting of fixes that have regressed as well as identify that fixes are now available.\n"); 609 printf("Files that exhibit known bugs may provide\n"); 610 printf(" rdar://whatever such that if they fail the rdar will get cited\n"); 611 printf("Files that are expected to fail to compile should provide, as their last token sequence,\n"); 612 printf(" error:\n"); 613 printf(" or error: substring to match.\n"); 614 printf("Files that are expected to produce a runtime error message should provide, as their last token sequence,\n"); 615 printf(" warning: string to match\n"); 616 printf("\n%s will compile and run all configurations of the test files and report a summary at the end. Good luck.\n", whoami); 617 printf(" Blaine Garst blaine@apple.com\n"); 618} 619 620int main(int argc, char *argv[]) { 621 printf("running on %s-bit architecture\n", sizeof(long) == 4 ? "32" : "64"); 622 char *compilerDir = "/usr/bin"; 623 NSMutableArray *generators = [NSMutableArray new]; 624 bool doFast = false; 625 bool doStdin = false; 626 bool onlyOpen = false; 627 char *libraryPath = getenv("DYLD_LIBRARY_PATH"); 628 char *frameworkPath = getenv("DYLD_FRAMEWORK_PATH"); 629 // process options 630 while (argc > 1) { 631 if (!strcmp(argv[1], "-fast")) { 632 doFast = true; 633 --argc; 634 ++argv; 635 } 636 else if (!strcmp(argv[1], "-dyld")) { 637 doFast = true; 638 --argc; 639 ++argv; 640 frameworkPath = argv[1]; 641 libraryPath = argv[1]; 642 --argc; 643 ++argv; 644 } 645 else if (!strcmp(argv[1], "-open")) { 646 onlyOpen = true; 647 --argc; 648 ++argv; 649 } 650 else if (!strcmp(argv[1], "-clang")) { 651 DoClang = true; 652 --argc; 653 ++argv; 654 } 655 else if (!strcmp(argv[1], "-e")) { 656 Everything = true; 657 --argc; 658 ++argv; 659 } 660 else if (!strcmp(argv[1], "--")) { 661 doStdin = true; 662 --argc; 663 ++argv; 664 } 665 else if (!strcmp(argv[1], "-")) { 666 help(argv[0]); 667 return 1; 668 } 669 else if (argc > 1 && isDirectory(argv[1])) { 670 compilerDir = argv[1]; 671 ++argv; 672 --argc; 673 } 674 else 675 break; 676 } 677 // process remaining arguments, or stdin 678 if (argc == 1) { 679 if (doStdin) 680 generators = (NSMutableArray *)[TestFileExeGenerator generatorsFromFILE:stdin]; 681 else { 682 help(argv[0]); 683 return 1; 684 } 685 } 686 else while (argc > 1) { 687 TestFileExeGenerator *generator = [TestFileExeGenerator generatorFromFilename:argv[1]]; 688 if (generator) [generators addObject:generator]; 689 ++argv; 690 --argc; 691 } 692 // see if we can generate all possibilities 693 NSMutableArray *failureToCompile = [NSMutableArray new]; 694 NSMutableArray *failureToFailToCompile = [NSMutableArray new]; 695 NSMutableArray *failureToRun = [NSMutableArray new]; 696 NSMutableArray *successes = [NSMutableArray new]; 697 for (TestFileExeGenerator *generator in generators) { 698 //NSLog(@"got %@", generator); 699 if (onlyOpen && !generator.open) { 700 //printf("skipping resolved test %s\n", generator.filename); 701 continue; // skip closed if onlyOpen 702 } 703 if (!onlyOpen && generator.open) { 704 //printf("skipping open test %s\n", generator.filename); 705 continue; // skip open if not asked for onlyOpen 706 } 707 generator.compilerPath = compilerDir; 708 NSArray *tests = [generator allLines]; 709 for (TestFileExe *line in tests) { 710 line.frameworkPath = frameworkPath; // tell generators about it instead XXX 711 line.libraryPath = libraryPath; // tell generators about it instead XXX 712 if ([line shouldFail]) { 713 if (doFast) continue; // don't recompile & don't count as success 714 if ([line compileWithExpectedFailure]) { 715 [successes addObject:line]; 716 } 717 else 718 [failureToFailToCompile addObject:line]; 719 } 720 else if ([line compileUnlessExists:doFast]) { 721 if ([line run]) { 722 printf("%s ran successfully\n", line.binaryName); 723 [successes addObject:line]; 724 } 725 else { 726 [failureToRun addObject:line]; 727 } 728 } 729 else { 730 [failureToCompile addObject:line]; 731 } 732 } 733 } 734 printf("\n--- results ---\n\n%lu successes\n%lu unexpected compile failures\n%lu failure to fail to compile errors\n%lu run failures\n", 735 [successes count], [failureToCompile count], [failureToFailToCompile count], [failureToRun count]); 736 printDetails(failureToCompile, "unexpected compile failures"); 737 printDetails(failureToFailToCompile, "should have failed to compile but didn't failures"); 738 printDetails(failureToRun, "run failures"); 739 740 if (onlyOpen && [successes count]) { 741 NSMutableSet *radars = [NSMutableSet new]; 742 printf("The following tests ran successfully suggesting that they are now resolved:\n"); 743 for (TestFileExe *line in successes) { 744 printf("%s\n", line.binaryName); 745 if (line.radar) [radars addObject:line.generator]; 746 } 747 if ([radars count]) { 748 printf("The following radars may be resolved:\n"); 749 for (TestFileExeGenerator *line in radars) { 750 printf("%s\n", line.radar); 751 } 752 } 753 } 754 755 return [failureToCompile count] + [failureToRun count]; 756} 757 758#include <sys/stat.h> 759 760static bool isDirectory(char *path) { 761 struct stat statb; 762 int retval = stat(path, &statb); 763 if (retval != 0) return false; 764 if (statb.st_mode & S_IFDIR) return true; 765 return false; 766} 767 768static bool isExecutable(char *path) { 769 struct stat statb; 770 int retval = stat(path, &statb); 771 if (retval != 0) return false; 772 if (!(statb.st_mode & S_IFREG)) return false; 773 if (statb.st_mode & S_IXUSR) return true; 774 return false; 775} 776 777static bool isYounger(char *source, char *binary) { 778 struct stat statb; 779 int retval = stat(binary, &statb); 780 if (retval != 0) return true; // if doesn't exit, lie 781 782 struct stat stata; 783 retval = stat(source, &stata); 784 if (retval != 0) return true; // we're hosed 785 // the greater the timeval the younger it is 786 if (stata.st_mtimespec.tv_sec > statb.st_mtimespec.tv_sec) return true; 787 if (stata.st_mtimespec.tv_nsec > statb.st_mtimespec.tv_nsec) return true; 788 return false; 789} 790 791static bool readErrorFile(char *buffer, const char *from) { 792 int fd = open(from, 0); 793 if (fd < 0) { 794 printf("didn't open %s, (might not have been created?)\n", buffer); 795 return false; 796 } 797 int count = read(fd, buffer, 512); 798 if (count < 1) { 799 printf("read error on %s\n", buffer); 800 return false; 801 } 802 buffer[count-1] = 0; // zap newline 803 return true; 804} 805