1#!/usr/bin/env python 2 3import argparse 4import sys 5import os 6 7from subprocess import call 8 9SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) 10PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") 11DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR, 12 os.path.pardir, 13 os.path.pardir, 14 os.path.pardir)) 15 16 17def add(parser, args): 18 import SATestAdd 19 from ProjectMap import ProjectInfo 20 21 if args.source == "git" and (args.origin == "" or args.commit == ""): 22 parser.error( 23 "Please provide both --origin and --commit if source is 'git'") 24 25 if args.source != "git" and (args.origin != "" or args.commit != ""): 26 parser.error("Options --origin and --commit don't make sense when " 27 "source is not 'git'") 28 29 project = ProjectInfo(args.name[0], args.mode, args.source, args.origin, 30 args.commit) 31 32 SATestAdd.add_new_project(project) 33 34 35def build(parser, args): 36 import SATestBuild 37 38 SATestBuild.VERBOSE = args.verbose 39 40 projects = get_projects(parser, args.projects) 41 tester = SATestBuild.RegressionTester(args.jobs, 42 projects, 43 args.override_compiler, 44 args.extra_analyzer_config, 45 args.regenerate, 46 args.strictness) 47 tests_passed = tester.test_all() 48 49 if not tests_passed: 50 sys.stderr.write("ERROR: Tests failed.\n") 51 sys.exit(42) 52 53 54def compare(parser, args): 55 import CmpRuns 56 57 choices = [CmpRuns.HistogramType.RELATIVE.value, 58 CmpRuns.HistogramType.LOG_RELATIVE.value, 59 CmpRuns.HistogramType.ABSOLUTE.value] 60 61 if args.histogram is not None and args.histogram not in choices: 62 parser.error("Incorrect histogram type, available choices are {}" 63 .format(choices)) 64 65 dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) 66 dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) 67 68 CmpRuns.dump_scan_build_results_diff(dir_old, dir_new, 69 show_stats=args.show_stats, 70 stats_only=args.stats_only, 71 histogram=args.histogram, 72 verbose_log=args.verbose_log) 73 74 75def update(parser, args): 76 import SATestUpdateDiffs 77 from ProjectMap import ProjectMap 78 79 project_map = ProjectMap() 80 for project in project_map.projects: 81 SATestUpdateDiffs.update_reference_results(project) 82 83 84def benchmark(parser, args): 85 from SATestBenchmark import Benchmark 86 87 projects = get_projects(parser, args.projects) 88 benchmark = Benchmark(projects, args.iterations, args.output) 89 benchmark.run() 90 91 92def benchmark_compare(parser, args): 93 import SATestBenchmark 94 SATestBenchmark.compare(args.old, args.new, args.output) 95 96 97def get_projects(parser, projects_str): 98 from ProjectMap import ProjectMap 99 100 project_map = ProjectMap() 101 projects = project_map.projects 102 103 if projects_str: 104 projects_arg = projects_str.split(",") 105 available_projects = [project.name 106 for project in projects] 107 108 # validate that given projects are present in the project map file 109 for manual_project in projects_arg: 110 if manual_project not in available_projects: 111 parser.error("Project '{project}' is not found in " 112 "the project map file. Available projects are " 113 "{all}.".format(project=manual_project, 114 all=available_projects)) 115 116 projects = [project.with_fields(enabled=project.name in projects_arg) 117 for project in projects] 118 119 return projects 120 121 122def docker(parser, args): 123 if len(args.rest) > 0: 124 if args.rest[0] != "--": 125 parser.error("REST arguments should start with '--'") 126 args.rest = args.rest[1:] 127 128 if args.build_image: 129 docker_build_image() 130 elif args.shell: 131 docker_shell(args) 132 else: 133 sys.exit(docker_run(args, ' '.join(args.rest))) 134 135 136def docker_build_image(): 137 sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), 138 shell=True)) 139 140 141def docker_shell(args): 142 try: 143 # First we need to start the docker container in a waiting mode, 144 # so it doesn't do anything, but most importantly keeps working 145 # while the shell session is in progress. 146 docker_run(args, "--wait", "--detach") 147 # Since the docker container is running, we can actually connect to it 148 call("docker exec -it satest bash", shell=True) 149 150 except KeyboardInterrupt: 151 pass 152 153 finally: 154 docker_cleanup() 155 156 157def docker_run(args, command, docker_args=""): 158 try: 159 return call("docker run --rm --name satest " 160 "-v {llvm}:/llvm-project " 161 "-v {build}:/build " 162 "-v {clang}:/analyzer " 163 "-v {scripts}:/scripts " 164 "-v {projects}:/projects " 165 "{docker_args} " 166 "satest-image:latest {command}" 167 .format(llvm=args.llvm_project_dir, 168 build=args.build_dir, 169 clang=args.clang_dir, 170 scripts=SCRIPTS_DIR, 171 projects=PROJECTS_DIR, 172 docker_args=docker_args, 173 command=command), 174 shell=True) 175 176 except KeyboardInterrupt: 177 docker_cleanup() 178 179 180def docker_cleanup(): 181 print("Please wait for docker to clean up") 182 call("docker stop satest", shell=True) 183 184 185def main(): 186 parser = argparse.ArgumentParser() 187 subparsers = parser.add_subparsers() 188 189 # add subcommand 190 add_parser = subparsers.add_parser( 191 "add", 192 help="Add a new project for the analyzer testing.") 193 # TODO: Add an option not to build. 194 # TODO: Set the path to the Repository directory. 195 add_parser.add_argument("name", nargs=1, help="Name of the new project") 196 add_parser.add_argument("--mode", action="store", default=1, type=int, 197 choices=[0, 1, 2], 198 help="Build mode: 0 for single file project, " 199 "1 for scan_build, " 200 "2 for single file c++11 project") 201 add_parser.add_argument("--source", action="store", default="script", 202 choices=["script", "git", "zip"], 203 help="Source type of the new project: " 204 "'git' for getting from git " 205 "(please provide --origin and --commit), " 206 "'zip' for unpacking source from a zip file, " 207 "'script' for downloading source by running " 208 "a custom script") 209 add_parser.add_argument("--origin", action="store", default="", 210 help="Origin link for a git repository") 211 add_parser.add_argument("--commit", action="store", default="", 212 help="Git hash for a commit to checkout") 213 add_parser.set_defaults(func=add) 214 215 # build subcommand 216 build_parser = subparsers.add_parser( 217 "build", 218 help="Build projects from the project map and compare results with " 219 "the reference.") 220 build_parser.add_argument("--strictness", dest="strictness", 221 type=int, default=0, 222 help="0 to fail on runtime errors, 1 to fail " 223 "when the number of found bugs are different " 224 "from the reference, 2 to fail on any " 225 "difference from the reference. Default is 0.") 226 build_parser.add_argument("-r", dest="regenerate", action="store_true", 227 default=False, 228 help="Regenerate reference output.") 229 build_parser.add_argument("--override-compiler", action="store_true", 230 default=False, help="Call scan-build with " 231 "--override-compiler option.") 232 build_parser.add_argument("-j", "--jobs", dest="jobs", 233 type=int, default=0, 234 help="Number of projects to test concurrently") 235 build_parser.add_argument("--extra-analyzer-config", 236 dest="extra_analyzer_config", type=str, 237 default="", 238 help="Arguments passed to to -analyzer-config") 239 build_parser.add_argument("--projects", action="store", default="", 240 help="Comma-separated list of projects to test") 241 build_parser.add_argument("-v", "--verbose", action="count", default=0) 242 build_parser.set_defaults(func=build) 243 244 # compare subcommand 245 cmp_parser = subparsers.add_parser( 246 "compare", 247 help="Comparing two static analyzer runs in terms of " 248 "reported warnings and execution time statistics.") 249 cmp_parser.add_argument("--root-old", dest="root_old", 250 help="Prefix to ignore on source files for " 251 "OLD directory", 252 action="store", type=str, default="") 253 cmp_parser.add_argument("--root-new", dest="root_new", 254 help="Prefix to ignore on source files for " 255 "NEW directory", 256 action="store", type=str, default="") 257 cmp_parser.add_argument("--verbose-log", dest="verbose_log", 258 help="Write additional information to LOG " 259 "[default=None]", 260 action="store", type=str, default=None, 261 metavar="LOG") 262 cmp_parser.add_argument("--stats-only", action="store_true", 263 dest="stats_only", default=False, 264 help="Only show statistics on reports") 265 cmp_parser.add_argument("--show-stats", action="store_true", 266 dest="show_stats", default=False, 267 help="Show change in statistics") 268 cmp_parser.add_argument("--histogram", action="store", default=None, 269 help="Show histogram of paths differences. " 270 "Requires matplotlib") 271 cmp_parser.add_argument("old", nargs=1, help="Directory with old results") 272 cmp_parser.add_argument("new", nargs=1, help="Directory with new results") 273 cmp_parser.set_defaults(func=compare) 274 275 # update subcommand 276 upd_parser = subparsers.add_parser( 277 "update", 278 help="Update static analyzer reference results based on the previous " 279 "run of SATest build. Assumes that SATest build was just run.") 280 # TODO: add option to decide whether we should use git 281 upd_parser.set_defaults(func=update) 282 283 # docker subcommand 284 dock_parser = subparsers.add_parser( 285 "docker", 286 help="Run regression system in the docker.") 287 288 dock_parser.add_argument("--build-image", action="store_true", 289 help="Build docker image for running tests.") 290 dock_parser.add_argument("--shell", action="store_true", 291 help="Start a shell on docker.") 292 dock_parser.add_argument("--llvm-project-dir", action="store", 293 default=DEFAULT_LLVM_DIR, 294 help="Path to LLVM source code. Defaults " 295 "to the repo where this script is located. ") 296 dock_parser.add_argument("--build-dir", action="store", default="", 297 help="Path to a directory where docker should " 298 "build LLVM code.") 299 dock_parser.add_argument("--clang-dir", action="store", default="", 300 help="Path to find/install LLVM installation.") 301 dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[], 302 help="Additionall args that will be forwarded " 303 "to the docker's entrypoint.") 304 dock_parser.set_defaults(func=docker) 305 306 # benchmark subcommand 307 bench_parser = subparsers.add_parser( 308 "benchmark", 309 help="Run benchmarks by building a set of projects multiple times.") 310 311 bench_parser.add_argument("-i", "--iterations", action="store", 312 type=int, default=20, 313 help="Number of iterations for building each " 314 "project.") 315 bench_parser.add_argument("-o", "--output", action="store", 316 default="benchmark.csv", 317 help="Output csv file for the benchmark results") 318 bench_parser.add_argument("--projects", action="store", default="", 319 help="Comma-separated list of projects to test") 320 bench_parser.set_defaults(func=benchmark) 321 322 bench_subparsers = bench_parser.add_subparsers() 323 bench_compare_parser = bench_subparsers.add_parser( 324 "compare", 325 help="Compare benchmark runs.") 326 bench_compare_parser.add_argument("--old", action="store", required=True, 327 help="Benchmark reference results to " 328 "compare agains.") 329 bench_compare_parser.add_argument("--new", action="store", required=True, 330 help="New benchmark results to check.") 331 bench_compare_parser.add_argument("-o", "--output", 332 action="store", required=True, 333 help="Output file for plots.") 334 bench_compare_parser.set_defaults(func=benchmark_compare) 335 336 args = parser.parse_args() 337 args.func(parser, args) 338 339 340if __name__ == "__main__": 341 main() 342