1#!/usr/bin/env python 2 3import argparse 4import sys 5import os 6 7from subprocess import call 8 9SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) 10PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") 11DEFAULT_LLVM_DIR = os.path.realpath( 12 os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir) 13) 14 15 16def add(parser, args): 17 import SATestAdd 18 from ProjectMap import ProjectInfo 19 20 if args.source == "git" and (args.origin == "" or args.commit == ""): 21 parser.error("Please provide both --origin and --commit if source is 'git'") 22 23 if args.source != "git" and (args.origin != "" or args.commit != ""): 24 parser.error( 25 "Options --origin and --commit don't make sense when " "source is not 'git'" 26 ) 27 28 project = ProjectInfo( 29 args.name[0], args.mode, args.source, args.origin, args.commit 30 ) 31 32 SATestAdd.add_new_project(project) 33 34 35def build(parser, args): 36 import SATestBuild 37 38 SATestBuild.VERBOSE = args.verbose 39 40 projects = get_projects(parser, args) 41 tester = SATestBuild.RegressionTester( 42 args.jobs, 43 projects, 44 args.override_compiler, 45 args.extra_analyzer_config, 46 args.extra_checkers, 47 args.regenerate, 48 args.strictness, 49 ) 50 tests_passed = tester.test_all() 51 52 if not tests_passed: 53 sys.stderr.write("ERROR: Tests failed.\n") 54 sys.exit(42) 55 56 57def compare(parser, args): 58 import CmpRuns 59 60 choices = [ 61 CmpRuns.HistogramType.RELATIVE.value, 62 CmpRuns.HistogramType.LOG_RELATIVE.value, 63 CmpRuns.HistogramType.ABSOLUTE.value, 64 ] 65 66 if args.histogram is not None and args.histogram not in choices: 67 parser.error( 68 "Incorrect histogram type, available choices are {}".format(choices) 69 ) 70 71 dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) 72 dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) 73 74 CmpRuns.dump_scan_build_results_diff( 75 dir_old, 76 dir_new, 77 show_stats=args.show_stats, 78 stats_only=args.stats_only, 79 histogram=args.histogram, 80 verbose_log=args.verbose_log, 81 ) 82 83 84def update(parser, args): 85 import SATestUpdateDiffs 86 from ProjectMap import ProjectMap 87 88 project_map = ProjectMap() 89 for project in project_map.projects: 90 SATestUpdateDiffs.update_reference_results(project, args.git) 91 92 93def benchmark(parser, args): 94 from SATestBenchmark import Benchmark 95 96 projects = get_projects(parser, args) 97 benchmark = Benchmark(projects, args.iterations, args.output) 98 benchmark.run() 99 100 101def benchmark_compare(parser, args): 102 import SATestBenchmark 103 104 SATestBenchmark.compare(args.old, args.new, args.output) 105 106 107def get_projects(parser, args): 108 from ProjectMap import ProjectMap, Size 109 110 project_map = ProjectMap() 111 projects = project_map.projects 112 113 def filter_projects(projects, predicate, force=False): 114 return [ 115 project.with_fields( 116 enabled=(force or project.enabled) and predicate(project) 117 ) 118 for project in projects 119 ] 120 121 if args.projects: 122 projects_arg = args.projects.split(",") 123 available_projects = [project.name for project in projects] 124 125 # validate that given projects are present in the project map file 126 for manual_project in projects_arg: 127 if manual_project not in available_projects: 128 parser.error( 129 "Project '{project}' is not found in " 130 "the project map file. Available projects are " 131 "{all}.".format(project=manual_project, all=available_projects) 132 ) 133 134 projects = filter_projects( 135 projects, lambda project: project.name in projects_arg, force=True 136 ) 137 138 try: 139 max_size = Size.from_str(args.max_size) 140 except ValueError as e: 141 parser.error("{}".format(e)) 142 143 projects = filter_projects(projects, lambda project: project.size <= max_size) 144 145 return projects 146 147 148def docker(parser, args): 149 if len(args.rest) > 0: 150 if args.rest[0] != "--": 151 parser.error("REST arguments should start with '--'") 152 args.rest = args.rest[1:] 153 154 if args.build_image: 155 docker_build_image() 156 elif args.shell: 157 docker_shell(args) 158 else: 159 sys.exit(docker_run(args, " ".join(args.rest))) 160 161 162def docker_build_image(): 163 sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True)) 164 165 166def docker_shell(args): 167 try: 168 # First we need to start the docker container in a waiting mode, 169 # so it doesn't do anything, but most importantly keeps working 170 # while the shell session is in progress. 171 docker_run(args, "--wait", "--detach") 172 # Since the docker container is running, we can actually connect to it 173 call("docker exec -it satest bash", shell=True) 174 175 except KeyboardInterrupt: 176 pass 177 178 finally: 179 docker_cleanup() 180 181 182def docker_run(args, command, docker_args=""): 183 try: 184 return call( 185 "docker run --rm --name satest " 186 "-v {llvm}:/llvm-project " 187 "-v {build}:/build " 188 "-v {clang}:/analyzer " 189 "-v {scripts}:/scripts " 190 "-v {projects}:/projects " 191 "{docker_args} " 192 "satest-image:latest {command}".format( 193 llvm=args.llvm_project_dir, 194 build=args.build_dir, 195 clang=args.clang_dir, 196 scripts=SCRIPTS_DIR, 197 projects=PROJECTS_DIR, 198 docker_args=docker_args, 199 command=command, 200 ), 201 shell=True, 202 ) 203 204 except KeyboardInterrupt: 205 docker_cleanup() 206 207 208def docker_cleanup(): 209 print("Please wait for docker to clean up") 210 call("docker stop satest", shell=True) 211 212 213def main(): 214 parser = argparse.ArgumentParser() 215 subparsers = parser.add_subparsers() 216 217 # add subcommand 218 add_parser = subparsers.add_parser( 219 "add", help="Add a new project for the analyzer testing." 220 ) 221 # TODO: Add an option not to build. 222 # TODO: Set the path to the Repository directory. 223 add_parser.add_argument("name", nargs=1, help="Name of the new project") 224 add_parser.add_argument( 225 "--mode", 226 action="store", 227 default=1, 228 type=int, 229 choices=[0, 1, 2], 230 help="Build mode: 0 for single file project, " 231 "1 for scan_build, " 232 "2 for single file c++11 project", 233 ) 234 add_parser.add_argument( 235 "--source", 236 action="store", 237 default="script", 238 choices=["script", "git", "zip"], 239 help="Source type of the new project: " 240 "'git' for getting from git " 241 "(please provide --origin and --commit), " 242 "'zip' for unpacking source from a zip file, " 243 "'script' for downloading source by running " 244 "a custom script", 245 ) 246 add_parser.add_argument( 247 "--origin", action="store", default="", help="Origin link for a git repository" 248 ) 249 add_parser.add_argument( 250 "--commit", action="store", default="", help="Git hash for a commit to checkout" 251 ) 252 add_parser.set_defaults(func=add) 253 254 # build subcommand 255 build_parser = subparsers.add_parser( 256 "build", 257 help="Build projects from the project map and compare results with " 258 "the reference.", 259 ) 260 build_parser.add_argument( 261 "--strictness", 262 dest="strictness", 263 type=int, 264 default=0, 265 help="0 to fail on runtime errors, 1 to fail " 266 "when the number of found bugs are different " 267 "from the reference, 2 to fail on any " 268 "difference from the reference. Default is 0.", 269 ) 270 build_parser.add_argument( 271 "-r", 272 dest="regenerate", 273 action="store_true", 274 default=False, 275 help="Regenerate reference output.", 276 ) 277 build_parser.add_argument( 278 "--override-compiler", 279 action="store_true", 280 default=False, 281 help="Call scan-build with " "--override-compiler option.", 282 ) 283 build_parser.add_argument( 284 "-j", 285 "--jobs", 286 dest="jobs", 287 type=int, 288 default=0, 289 help="Number of projects to test concurrently", 290 ) 291 build_parser.add_argument( 292 "--extra-analyzer-config", 293 dest="extra_analyzer_config", 294 type=str, 295 default="", 296 help="Arguments passed to to -analyzer-config", 297 ) 298 build_parser.add_argument( 299 "--extra-checkers", 300 dest="extra_checkers", 301 type=str, 302 default="", 303 help="Extra checkers to enable", 304 ) 305 build_parser.add_argument( 306 "--projects", 307 action="store", 308 default="", 309 help="Comma-separated list of projects to test", 310 ) 311 build_parser.add_argument( 312 "--max-size", 313 action="store", 314 default=None, 315 help="Maximum size for the projects to test", 316 ) 317 build_parser.add_argument("-v", "--verbose", action="count", default=0) 318 build_parser.set_defaults(func=build) 319 320 # compare subcommand 321 cmp_parser = subparsers.add_parser( 322 "compare", 323 help="Comparing two static analyzer runs in terms of " 324 "reported warnings and execution time statistics.", 325 ) 326 cmp_parser.add_argument( 327 "--root-old", 328 dest="root_old", 329 help="Prefix to ignore on source files for " "OLD directory", 330 action="store", 331 type=str, 332 default="", 333 ) 334 cmp_parser.add_argument( 335 "--root-new", 336 dest="root_new", 337 help="Prefix to ignore on source files for " "NEW directory", 338 action="store", 339 type=str, 340 default="", 341 ) 342 cmp_parser.add_argument( 343 "--verbose-log", 344 dest="verbose_log", 345 help="Write additional information to LOG " "[default=None]", 346 action="store", 347 type=str, 348 default=None, 349 metavar="LOG", 350 ) 351 cmp_parser.add_argument( 352 "--stats-only", 353 action="store_true", 354 dest="stats_only", 355 default=False, 356 help="Only show statistics on reports", 357 ) 358 cmp_parser.add_argument( 359 "--show-stats", 360 action="store_true", 361 dest="show_stats", 362 default=False, 363 help="Show change in statistics", 364 ) 365 cmp_parser.add_argument( 366 "--histogram", 367 action="store", 368 default=None, 369 help="Show histogram of paths differences. " "Requires matplotlib", 370 ) 371 cmp_parser.add_argument("old", nargs=1, help="Directory with old results") 372 cmp_parser.add_argument("new", nargs=1, help="Directory with new results") 373 cmp_parser.set_defaults(func=compare) 374 375 # update subcommand 376 upd_parser = subparsers.add_parser( 377 "update", 378 help="Update static analyzer reference results based on the previous " 379 "run of SATest build. Assumes that SATest build was just run.", 380 ) 381 upd_parser.add_argument( 382 "--git", action="store_true", help="Stage updated results using git." 383 ) 384 upd_parser.set_defaults(func=update) 385 386 # docker subcommand 387 dock_parser = subparsers.add_parser( 388 "docker", help="Run regression system in the docker." 389 ) 390 391 dock_parser.add_argument( 392 "--build-image", 393 action="store_true", 394 help="Build docker image for running tests.", 395 ) 396 dock_parser.add_argument( 397 "--shell", action="store_true", help="Start a shell on docker." 398 ) 399 dock_parser.add_argument( 400 "--llvm-project-dir", 401 action="store", 402 default=DEFAULT_LLVM_DIR, 403 help="Path to LLVM source code. Defaults " 404 "to the repo where this script is located. ", 405 ) 406 dock_parser.add_argument( 407 "--build-dir", 408 action="store", 409 default="", 410 help="Path to a directory where docker should " "build LLVM code.", 411 ) 412 dock_parser.add_argument( 413 "--clang-dir", 414 action="store", 415 default="", 416 help="Path to find/install LLVM installation.", 417 ) 418 dock_parser.add_argument( 419 "rest", 420 nargs=argparse.REMAINDER, 421 default=[], 422 help="Additional args that will be forwarded " "to the docker's entrypoint.", 423 ) 424 dock_parser.set_defaults(func=docker) 425 426 # benchmark subcommand 427 bench_parser = subparsers.add_parser( 428 "benchmark", help="Run benchmarks by building a set of projects multiple times." 429 ) 430 431 bench_parser.add_argument( 432 "-i", 433 "--iterations", 434 action="store", 435 type=int, 436 default=20, 437 help="Number of iterations for building each " "project.", 438 ) 439 bench_parser.add_argument( 440 "-o", 441 "--output", 442 action="store", 443 default="benchmark.csv", 444 help="Output csv file for the benchmark results", 445 ) 446 bench_parser.add_argument( 447 "--projects", 448 action="store", 449 default="", 450 help="Comma-separated list of projects to test", 451 ) 452 bench_parser.add_argument( 453 "--max-size", 454 action="store", 455 default=None, 456 help="Maximum size for the projects to test", 457 ) 458 bench_parser.set_defaults(func=benchmark) 459 460 bench_subparsers = bench_parser.add_subparsers() 461 bench_compare_parser = bench_subparsers.add_parser( 462 "compare", help="Compare benchmark runs." 463 ) 464 bench_compare_parser.add_argument( 465 "--old", 466 action="store", 467 required=True, 468 help="Benchmark reference results to " "compare agains.", 469 ) 470 bench_compare_parser.add_argument( 471 "--new", action="store", required=True, help="New benchmark results to check." 472 ) 473 bench_compare_parser.add_argument( 474 "-o", "--output", action="store", required=True, help="Output file for plots." 475 ) 476 bench_compare_parser.set_defaults(func=benchmark_compare) 477 478 args = parser.parse_args() 479 args.func(parser, args) 480 481 482if __name__ == "__main__": 483 main() 484