1#!/usr/bin/env python 2 3import argparse 4import sys 5import os 6 7from subprocess import call 8 9SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) 10PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") 11DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR, 12 os.path.pardir, 13 os.path.pardir, 14 os.path.pardir)) 15 16 17def add(parser, args): 18 import SATestAdd 19 from ProjectMap import ProjectInfo 20 21 if args.source == "git" and (args.origin == "" or args.commit == ""): 22 parser.error( 23 "Please provide both --origin and --commit if source is 'git'") 24 25 if args.source != "git" and (args.origin != "" or args.commit != ""): 26 parser.error("Options --origin and --commit don't make sense when " 27 "source is not 'git'") 28 29 project = ProjectInfo(args.name[0], args.mode, args.source, args.origin, 30 args.commit) 31 32 SATestAdd.add_new_project(project) 33 34 35def build(parser, args): 36 import SATestBuild 37 38 SATestBuild.VERBOSE = args.verbose 39 40 projects = get_projects(parser, args) 41 tester = SATestBuild.RegressionTester(args.jobs, 42 projects, 43 args.override_compiler, 44 args.extra_analyzer_config, 45 args.extra_checkers, 46 args.regenerate, 47 args.strictness) 48 tests_passed = tester.test_all() 49 50 if not tests_passed: 51 sys.stderr.write("ERROR: Tests failed.\n") 52 sys.exit(42) 53 54 55def compare(parser, args): 56 import CmpRuns 57 58 choices = [CmpRuns.HistogramType.RELATIVE.value, 59 CmpRuns.HistogramType.LOG_RELATIVE.value, 60 CmpRuns.HistogramType.ABSOLUTE.value] 61 62 if args.histogram is not None and args.histogram not in choices: 63 parser.error("Incorrect histogram type, available choices are {}" 64 .format(choices)) 65 66 dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) 67 dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) 68 69 CmpRuns.dump_scan_build_results_diff(dir_old, dir_new, 70 show_stats=args.show_stats, 71 stats_only=args.stats_only, 72 histogram=args.histogram, 73 verbose_log=args.verbose_log) 74 75 76def update(parser, args): 77 import SATestUpdateDiffs 78 from ProjectMap import ProjectMap 79 80 project_map = ProjectMap() 81 for project in project_map.projects: 82 SATestUpdateDiffs.update_reference_results(project, args.git) 83 84 85def benchmark(parser, args): 86 from SATestBenchmark import Benchmark 87 88 projects = get_projects(parser, args) 89 benchmark = Benchmark(projects, args.iterations, args.output) 90 benchmark.run() 91 92 93def benchmark_compare(parser, args): 94 import SATestBenchmark 95 SATestBenchmark.compare(args.old, args.new, args.output) 96 97 98def get_projects(parser, args): 99 from ProjectMap import ProjectMap, Size 100 101 project_map = ProjectMap() 102 projects = project_map.projects 103 104 def filter_projects(projects, predicate, force=False): 105 return [project.with_fields(enabled=(force or project.enabled) and 106 predicate(project)) 107 for project in projects] 108 109 if args.projects: 110 projects_arg = args.projects.split(",") 111 available_projects = [project.name 112 for project in projects] 113 114 # validate that given projects are present in the project map file 115 for manual_project in projects_arg: 116 if manual_project not in available_projects: 117 parser.error("Project '{project}' is not found in " 118 "the project map file. Available projects are " 119 "{all}.".format(project=manual_project, 120 all=available_projects)) 121 122 projects = filter_projects(projects, lambda project: 123 project.name in projects_arg, 124 force=True) 125 126 try: 127 max_size = Size.from_str(args.max_size) 128 except ValueError as e: 129 parser.error("{}".format(e)) 130 131 projects = filter_projects(projects, lambda project: 132 project.size <= max_size) 133 134 return projects 135 136 137def docker(parser, args): 138 if len(args.rest) > 0: 139 if args.rest[0] != "--": 140 parser.error("REST arguments should start with '--'") 141 args.rest = args.rest[1:] 142 143 if args.build_image: 144 docker_build_image() 145 elif args.shell: 146 docker_shell(args) 147 else: 148 sys.exit(docker_run(args, ' '.join(args.rest))) 149 150 151def docker_build_image(): 152 sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), 153 shell=True)) 154 155 156def docker_shell(args): 157 try: 158 # First we need to start the docker container in a waiting mode, 159 # so it doesn't do anything, but most importantly keeps working 160 # while the shell session is in progress. 161 docker_run(args, "--wait", "--detach") 162 # Since the docker container is running, we can actually connect to it 163 call("docker exec -it satest bash", shell=True) 164 165 except KeyboardInterrupt: 166 pass 167 168 finally: 169 docker_cleanup() 170 171 172def docker_run(args, command, docker_args=""): 173 try: 174 return call("docker run --rm --name satest " 175 "-v {llvm}:/llvm-project " 176 "-v {build}:/build " 177 "-v {clang}:/analyzer " 178 "-v {scripts}:/scripts " 179 "-v {projects}:/projects " 180 "{docker_args} " 181 "satest-image:latest {command}" 182 .format(llvm=args.llvm_project_dir, 183 build=args.build_dir, 184 clang=args.clang_dir, 185 scripts=SCRIPTS_DIR, 186 projects=PROJECTS_DIR, 187 docker_args=docker_args, 188 command=command), 189 shell=True) 190 191 except KeyboardInterrupt: 192 docker_cleanup() 193 194 195def docker_cleanup(): 196 print("Please wait for docker to clean up") 197 call("docker stop satest", shell=True) 198 199 200def main(): 201 parser = argparse.ArgumentParser() 202 subparsers = parser.add_subparsers() 203 204 # add subcommand 205 add_parser = subparsers.add_parser( 206 "add", 207 help="Add a new project for the analyzer testing.") 208 # TODO: Add an option not to build. 209 # TODO: Set the path to the Repository directory. 210 add_parser.add_argument("name", nargs=1, help="Name of the new project") 211 add_parser.add_argument("--mode", action="store", default=1, type=int, 212 choices=[0, 1, 2], 213 help="Build mode: 0 for single file project, " 214 "1 for scan_build, " 215 "2 for single file c++11 project") 216 add_parser.add_argument("--source", action="store", default="script", 217 choices=["script", "git", "zip"], 218 help="Source type of the new project: " 219 "'git' for getting from git " 220 "(please provide --origin and --commit), " 221 "'zip' for unpacking source from a zip file, " 222 "'script' for downloading source by running " 223 "a custom script") 224 add_parser.add_argument("--origin", action="store", default="", 225 help="Origin link for a git repository") 226 add_parser.add_argument("--commit", action="store", default="", 227 help="Git hash for a commit to checkout") 228 add_parser.set_defaults(func=add) 229 230 # build subcommand 231 build_parser = subparsers.add_parser( 232 "build", 233 help="Build projects from the project map and compare results with " 234 "the reference.") 235 build_parser.add_argument("--strictness", dest="strictness", 236 type=int, default=0, 237 help="0 to fail on runtime errors, 1 to fail " 238 "when the number of found bugs are different " 239 "from the reference, 2 to fail on any " 240 "difference from the reference. Default is 0.") 241 build_parser.add_argument("-r", dest="regenerate", action="store_true", 242 default=False, 243 help="Regenerate reference output.") 244 build_parser.add_argument("--override-compiler", action="store_true", 245 default=False, help="Call scan-build with " 246 "--override-compiler option.") 247 build_parser.add_argument("-j", "--jobs", dest="jobs", 248 type=int, default=0, 249 help="Number of projects to test concurrently") 250 build_parser.add_argument("--extra-analyzer-config", 251 dest="extra_analyzer_config", type=str, 252 default="", 253 help="Arguments passed to to -analyzer-config") 254 build_parser.add_argument("--extra-checkers", 255 dest="extra_checkers", type=str, 256 default="", 257 help="Extra checkers to enable") 258 build_parser.add_argument("--projects", action="store", default="", 259 help="Comma-separated list of projects to test") 260 build_parser.add_argument("--max-size", action="store", default=None, 261 help="Maximum size for the projects to test") 262 build_parser.add_argument("-v", "--verbose", action="count", default=0) 263 build_parser.set_defaults(func=build) 264 265 # compare subcommand 266 cmp_parser = subparsers.add_parser( 267 "compare", 268 help="Comparing two static analyzer runs in terms of " 269 "reported warnings and execution time statistics.") 270 cmp_parser.add_argument("--root-old", dest="root_old", 271 help="Prefix to ignore on source files for " 272 "OLD directory", 273 action="store", type=str, default="") 274 cmp_parser.add_argument("--root-new", dest="root_new", 275 help="Prefix to ignore on source files for " 276 "NEW directory", 277 action="store", type=str, default="") 278 cmp_parser.add_argument("--verbose-log", dest="verbose_log", 279 help="Write additional information to LOG " 280 "[default=None]", 281 action="store", type=str, default=None, 282 metavar="LOG") 283 cmp_parser.add_argument("--stats-only", action="store_true", 284 dest="stats_only", default=False, 285 help="Only show statistics on reports") 286 cmp_parser.add_argument("--show-stats", action="store_true", 287 dest="show_stats", default=False, 288 help="Show change in statistics") 289 cmp_parser.add_argument("--histogram", action="store", default=None, 290 help="Show histogram of paths differences. " 291 "Requires matplotlib") 292 cmp_parser.add_argument("old", nargs=1, help="Directory with old results") 293 cmp_parser.add_argument("new", nargs=1, help="Directory with new results") 294 cmp_parser.set_defaults(func=compare) 295 296 # update subcommand 297 upd_parser = subparsers.add_parser( 298 "update", 299 help="Update static analyzer reference results based on the previous " 300 "run of SATest build. Assumes that SATest build was just run.") 301 upd_parser.add_argument("--git", action="store_true", 302 help="Stage updated results using git.") 303 upd_parser.set_defaults(func=update) 304 305 # docker subcommand 306 dock_parser = subparsers.add_parser( 307 "docker", 308 help="Run regression system in the docker.") 309 310 dock_parser.add_argument("--build-image", action="store_true", 311 help="Build docker image for running tests.") 312 dock_parser.add_argument("--shell", action="store_true", 313 help="Start a shell on docker.") 314 dock_parser.add_argument("--llvm-project-dir", action="store", 315 default=DEFAULT_LLVM_DIR, 316 help="Path to LLVM source code. Defaults " 317 "to the repo where this script is located. ") 318 dock_parser.add_argument("--build-dir", action="store", default="", 319 help="Path to a directory where docker should " 320 "build LLVM code.") 321 dock_parser.add_argument("--clang-dir", action="store", default="", 322 help="Path to find/install LLVM installation.") 323 dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[], 324 help="Additional args that will be forwarded " 325 "to the docker's entrypoint.") 326 dock_parser.set_defaults(func=docker) 327 328 # benchmark subcommand 329 bench_parser = subparsers.add_parser( 330 "benchmark", 331 help="Run benchmarks by building a set of projects multiple times.") 332 333 bench_parser.add_argument("-i", "--iterations", action="store", 334 type=int, default=20, 335 help="Number of iterations for building each " 336 "project.") 337 bench_parser.add_argument("-o", "--output", action="store", 338 default="benchmark.csv", 339 help="Output csv file for the benchmark results") 340 bench_parser.add_argument("--projects", action="store", default="", 341 help="Comma-separated list of projects to test") 342 bench_parser.add_argument("--max-size", action="store", default=None, 343 help="Maximum size for the projects to test") 344 bench_parser.set_defaults(func=benchmark) 345 346 bench_subparsers = bench_parser.add_subparsers() 347 bench_compare_parser = bench_subparsers.add_parser( 348 "compare", 349 help="Compare benchmark runs.") 350 bench_compare_parser.add_argument("--old", action="store", required=True, 351 help="Benchmark reference results to " 352 "compare agains.") 353 bench_compare_parser.add_argument("--new", action="store", required=True, 354 help="New benchmark results to check.") 355 bench_compare_parser.add_argument("-o", "--output", 356 action="store", required=True, 357 help="Output file for plots.") 358 bench_compare_parser.set_defaults(func=benchmark_compare) 359 360 args = parser.parse_args() 361 args.func(parser, args) 362 363 364if __name__ == "__main__": 365 main() 366