utils/analyzer/SATest.py

#!/usr/bin/env python

import argparse
import sys
import os

from subprocess import call

SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
DEFAULT_LLVM_DIR = os.path.realpath(
    os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir)
)


def add(parser, args):
    import SATestAdd
    from ProjectMap import ProjectInfo

    if args.source == "git" and (args.origin == "" or args.commit == ""):
        parser.error("Please provide both --origin and --commit if source is 'git'")

    if args.source != "git" and (args.origin != "" or args.commit != ""):
        parser.error(
            "Options --origin and --commit don't make sense when " "source is not 'git'"
        )

    project = ProjectInfo(
        args.name[0], args.mode, args.source, args.origin, args.commit
    )

    SATestAdd.add_new_project(project)


def build(parser, args):
    import SATestBuild

    SATestBuild.VERBOSE = args.verbose

    projects = get_projects(parser, args)
    tester = SATestBuild.RegressionTester(
        args.jobs,
        projects,
        args.override_compiler,
        args.extra_analyzer_config,
        args.extra_checkers,
        args.regenerate,
        args.strictness,
    )
    tests_passed = tester.test_all()

    if not tests_passed:
        sys.stderr.write("ERROR: Tests failed.\n")
        sys.exit(42)


def compare(parser, args):
    import CmpRuns

    choices = [
        CmpRuns.HistogramType.RELATIVE.value,
        CmpRuns.HistogramType.LOG_RELATIVE.value,
        CmpRuns.HistogramType.ABSOLUTE.value,
    ]

    if args.histogram is not None and args.histogram not in choices:
        parser.error(
            "Incorrect histogram type, available choices are {}".format(choices)
        )

    dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
    dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)

    CmpRuns.dump_scan_build_results_diff(
        dir_old,
        dir_new,
        show_stats=args.show_stats,
        stats_only=args.stats_only,
        histogram=args.histogram,
        verbose_log=args.verbose_log,
    )


def update(parser, args):
    import SATestUpdateDiffs
    from ProjectMap import ProjectMap

    project_map = ProjectMap()
    for project in project_map.projects:
        SATestUpdateDiffs.update_reference_results(project, args.git)


def benchmark(parser, args):
    from SATestBenchmark import Benchmark

    projects = get_projects(parser, args)
    benchmark = Benchmark(projects, args.iterations, args.output)
    benchmark.run()


def benchmark_compare(parser, args):
    import SATestBenchmark

    SATestBenchmark.compare(args.old, args.new, args.output)


def get_projects(parser, args):
    from ProjectMap import ProjectMap, Size

    project_map = ProjectMap()
    projects = project_map.projects

    def filter_projects(projects, predicate, force=False):
        return [
            project.with_fields(
                enabled=(force or project.enabled) and predicate(project)
            )
            for project in projects
        ]

    if args.projects:
        projects_arg = args.projects.split(",")
        available_projects = [project.name for project in projects]

        # validate that given projects are present in the project map file
        for manual_project in projects_arg:
            if manual_project not in available_projects:
                parser.error(
                    "Project '{project}' is not found in "
                    "the project map file. Available projects are "
                    "{all}.".format(project=manual_project, all=available_projects)
                )

        projects = filter_projects(
            projects, lambda project: project.name in projects_arg, force=True
        )

    try:
        max_size = Size.from_str(args.max_size)
    except ValueError as e:
        parser.error("{}".format(e))

    projects = filter_projects(projects, lambda project: project.size <= max_size)

    return projects


def docker(parser, args):
    if len(args.rest) > 0:
        if args.rest[0] != "--":
            parser.error("REST arguments should start with '--'")
        args.rest = args.rest[1:]

    if args.build_image:
        docker_build_image()
    elif args.shell:
        docker_shell(args)
    else:
        sys.exit(docker_run(args, " ".join(args.rest)))


def docker_build_image():
    sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True))


def docker_shell(args):
    try:
        # First we need to start the docker container in a waiting mode,
        # so it doesn't do anything, but most importantly keeps working
        # while the shell session is in progress.
        docker_run(args, "--wait", "--detach")
        # Since the docker container is running, we can actually connect to it
        call("docker exec -it satest bash", shell=True)

    except KeyboardInterrupt:
        pass

    finally:
        docker_cleanup()


def docker_run(args, command, docker_args=""):
    try:
        return call(
            "docker run --rm --name satest "
            "-v {llvm}:/llvm-project "
            "-v {build}:/build "
            "-v {clang}:/analyzer "
            "-v {scripts}:/scripts "
            "-v {projects}:/projects "
            "{docker_args} "
            "satest-image:latest {command}".format(
                llvm=args.llvm_project_dir,
                build=args.build_dir,
                clang=args.clang_dir,
                scripts=SCRIPTS_DIR,
                projects=PROJECTS_DIR,
                docker_args=docker_args,
                command=command,
            ),
            shell=True,
        )

    except KeyboardInterrupt:
        docker_cleanup()


def docker_cleanup():
    print("Please wait for docker to clean up")
    call("docker stop satest", shell=True)


def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    # add subcommand
    add_parser = subparsers.add_parser(
        "add", help="Add a new project for the analyzer testing."
    )
    # TODO: Add an option not to build.
    # TODO: Set the path to the Repository directory.
    add_parser.add_argument("name", nargs=1, help="Name of the new project")
    add_parser.add_argument(
        "--mode",
        action="store",
        default=1,
        type=int,
        choices=[0, 1, 2],
        help="Build mode: 0 for single file project, "
        "1 for scan_build, "
        "2 for single file c++11 project",
    )
    add_parser.add_argument(
        "--source",
        action="store",
        default="script",
        choices=["script", "git", "zip"],
        help="Source type of the new project: "
        "'git' for getting from git "
        "(please provide --origin and --commit), "
        "'zip' for unpacking source from a zip file, "
        "'script' for downloading source by running "
        "a custom script",
    )
    add_parser.add_argument(
        "--origin", action="store", default="", help="Origin link for a git repository"
    )
    add_parser.add_argument(
        "--commit", action="store", default="", help="Git hash for a commit to checkout"
    )
    add_parser.set_defaults(func=add)

    # build subcommand
    build_parser = subparsers.add_parser(
        "build",
        help="Build projects from the project map and compare results with "
        "the reference.",
    )
    build_parser.add_argument(
        "--strictness",
        dest="strictness",
        type=int,
        default=0,
        help="0 to fail on runtime errors, 1 to fail "
        "when the number of found bugs are different "
        "from the reference, 2 to fail on any "
        "difference from the reference. Default is 0.",
    )
    build_parser.add_argument(
        "-r",
        dest="regenerate",
        action="store_true",
        default=False,
        help="Regenerate reference output.",
    )
    build_parser.add_argument(
        "--override-compiler",
        action="store_true",
        default=False,
        help="Call scan-build with " "--override-compiler option.",
    )
    build_parser.add_argument(
        "-j",
        "--jobs",
        dest="jobs",
        type=int,
        default=0,
        help="Number of projects to test concurrently",
    )
    build_parser.add_argument(
        "--extra-analyzer-config",
        dest="extra_analyzer_config",
        type=str,
        default="",
        help="Arguments passed to to -analyzer-config",
    )
    build_parser.add_argument(
        "--extra-checkers",
        dest="extra_checkers",
        type=str,
        default="",
        help="Extra checkers to enable",
    )
    build_parser.add_argument(
        "--projects",
        action="store",
        default="",
        help="Comma-separated list of projects to test",
    )
    build_parser.add_argument(
        "--max-size",
        action="store",
        default=None,
        help="Maximum size for the projects to test",
    )
    build_parser.add_argument("-v", "--verbose", action="count", default=0)
    build_parser.set_defaults(func=build)

    # compare subcommand
    cmp_parser = subparsers.add_parser(
        "compare",
        help="Comparing two static analyzer runs in terms of "
        "reported warnings and execution time statistics.",
    )
    cmp_parser.add_argument(
        "--root-old",
        dest="root_old",
        help="Prefix to ignore on source files for " "OLD directory",
        action="store",
        type=str,
        default="",
    )
    cmp_parser.add_argument(
        "--root-new",
        dest="root_new",
        help="Prefix to ignore on source files for " "NEW directory",
        action="store",
        type=str,
        default="",
    )
    cmp_parser.add_argument(
        "--verbose-log",
        dest="verbose_log",
        help="Write additional information to LOG " "[default=None]",
        action="store",
        type=str,
        default=None,
        metavar="LOG",
    )
    cmp_parser.add_argument(
        "--stats-only",
        action="store_true",
        dest="stats_only",
        default=False,
        help="Only show statistics on reports",
    )
    cmp_parser.add_argument(
        "--show-stats",
        action="store_true",
        dest="show_stats",
        default=False,
        help="Show change in statistics",
    )
    cmp_parser.add_argument(
        "--histogram",
        action="store",
        default=None,
        help="Show histogram of paths differences. " "Requires matplotlib",
    )
    cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
    cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
    cmp_parser.set_defaults(func=compare)

    # update subcommand
    upd_parser = subparsers.add_parser(
        "update",
        help="Update static analyzer reference results based on the previous "
        "run of SATest build. Assumes that SATest build was just run.",
    )
    upd_parser.add_argument(
        "--git", action="store_true", help="Stage updated results using git."
    )
    upd_parser.set_defaults(func=update)

    # docker subcommand
    dock_parser = subparsers.add_parser(
        "docker", help="Run regression system in the docker."
    )

    dock_parser.add_argument(
        "--build-image",
        action="store_true",
        help="Build docker image for running tests.",
    )
    dock_parser.add_argument(
        "--shell", action="store_true", help="Start a shell on docker."
    )
    dock_parser.add_argument(
        "--llvm-project-dir",
        action="store",
        default=DEFAULT_LLVM_DIR,
        help="Path to LLVM source code. Defaults "
        "to the repo where this script is located. ",
    )
    dock_parser.add_argument(
        "--build-dir",
        action="store",
        default="",
        help="Path to a directory where docker should " "build LLVM code.",
    )
    dock_parser.add_argument(
        "--clang-dir",
        action="store",
        default="",
        help="Path to find/install LLVM installation.",
    )
    dock_parser.add_argument(
        "rest",
        nargs=argparse.REMAINDER,
        default=[],
        help="Additional args that will be forwarded " "to the docker's entrypoint.",
    )
    dock_parser.set_defaults(func=docker)

    # benchmark subcommand
    bench_parser = subparsers.add_parser(
        "benchmark", help="Run benchmarks by building a set of projects multiple times."
    )

    bench_parser.add_argument(
        "-i",
        "--iterations",
        action="store",
        type=int,
        default=20,
        help="Number of iterations for building each " "project.",
    )
    bench_parser.add_argument(
        "-o",
        "--output",
        action="store",
        default="benchmark.csv",
        help="Output csv file for the benchmark results",
    )
    bench_parser.add_argument(
        "--projects",
        action="store",
        default="",
        help="Comma-separated list of projects to test",
    )
    bench_parser.add_argument(
        "--max-size",
        action="store",
        default=None,
        help="Maximum size for the projects to test",
    )
    bench_parser.set_defaults(func=benchmark)

    bench_subparsers = bench_parser.add_subparsers()
    bench_compare_parser = bench_subparsers.add_parser(
        "compare", help="Compare benchmark runs."
    )
    bench_compare_parser.add_argument(
        "--old",
        action="store",
        required=True,
        help="Benchmark reference results to " "compare agains.",
    )
    bench_compare_parser.add_argument(
        "--new", action="store", required=True, help="New benchmark results to check."
    )
    bench_compare_parser.add_argument(
        "-o", "--output", action="store", required=True, help="Output file for plots."
    )
    bench_compare_parser.set_defaults(func=benchmark_compare)

    args = parser.parse_args()
    args.func(parser, args)


if __name__ == "__main__":
    main()