xref: /llvm-project/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py (revision bc74625f50e216edd16f436c4fc81ff585b6c4c7) 
1#!/usr/bin/env python3
2#
3# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
4#
5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6# See https://llvm.org/LICENSE.txt for license information.
7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8#
9# ===-----------------------------------------------------------------------===#
10
11r"""
12ClangTidy Diff Checker
13======================
14
15This script reads input from a unified diff, runs clang-tidy on all changed
16files and outputs clang-tidy warnings in changed lines only. This is useful to
17detect clang-tidy regressions in the lines touched by a specific patch.
18Example usage for git/svn users:
19
20  git diff -U0 HEAD^ | clang-tidy-diff.py -p1
21  svn diff --diff-cmd=diff -x-U0 | \
22      clang-tidy-diff.py -fix -checks=-*,modernize-use-override
23
24"""
25
26import argparse
27import glob
28import json
29import multiprocessing
30import os
31import re
32import shutil
33import subprocess
34import sys
35import tempfile
36import threading
37import traceback
38from pathlib import Path
39
40try:
41    import yaml
42except ImportError:
43    yaml = None
44
45is_py2 = sys.version[0] == "2"
46
47if is_py2:
48    import Queue as queue
49else:
50    import queue as queue
51
52
53def run_tidy(task_queue, lock, timeout, failed_files):
54    watchdog = None
55    while True:
56        command = task_queue.get()
57        try:
58            proc = subprocess.Popen(
59                command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
60            )
61
62            if timeout is not None:
63                watchdog = threading.Timer(timeout, proc.kill)
64                watchdog.start()
65
66            stdout, stderr = proc.communicate()
67            if proc.returncode != 0:
68                if proc.returncode < 0:
69                    msg = "Terminated by signal %d : %s\n" % (
70                        -proc.returncode,
71                        " ".join(command),
72                    )
73                    stderr += msg.encode("utf-8")
74                failed_files.append(command)
75
76            with lock:
77                sys.stdout.write(stdout.decode("utf-8") + "\n")
78                sys.stdout.flush()
79                if stderr:
80                    sys.stderr.write(stderr.decode("utf-8") + "\n")
81                    sys.stderr.flush()
82        except Exception as e:
83            with lock:
84                sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")
85        finally:
86            with lock:
87                if not (timeout is None or watchdog is None):
88                    if not watchdog.is_alive():
89                        sys.stderr.write(
90                            "Terminated by timeout: " + " ".join(command) + "\n"
91                        )
92                    watchdog.cancel()
93            task_queue.task_done()
94
95
96def start_workers(max_tasks, tidy_caller, arguments):
97    for _ in range(max_tasks):
98        t = threading.Thread(target=tidy_caller, args=arguments)
99        t.daemon = True
100        t.start()
101
102
103def merge_replacement_files(tmpdir, mergefile):
104    """Merge all replacement files in a directory into a single file"""
105    # The fixes suggested by clang-tidy >= 4.0.0 are given under
106    # the top level key 'Diagnostics' in the output yaml files
107    mergekey = "Diagnostics"
108    merged = []
109    for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
110        content = yaml.safe_load(open(replacefile, "r"))
111        if not content:
112            continue  # Skip empty files.
113        merged.extend(content.get(mergekey, []))
114
115    if merged:
116        # MainSourceFile: The key is required by the definition inside
117        # include/clang/Tooling/ReplacementsYaml.h, but the value
118        # is actually never used inside clang-apply-replacements,
119        # so we set it to '' here.
120        output = {"MainSourceFile": "", mergekey: merged}
121        with open(mergefile, "w") as out:
122            yaml.safe_dump(output, out)
123    else:
124        # Empty the file:
125        open(mergefile, "w").close()
126
127
128def get_compiling_files(args):
129    """Read a compile_commands.json database and return a set of file paths"""
130    current_dir = Path.cwd()
131    compile_commands_json = (
132        (current_dir / args.build_path) if args.build_path else current_dir
133    )
134    compile_commands_json = compile_commands_json / "compile_commands.json"
135    files = set()
136    with open(compile_commands_json) as db_file:
137        db_json = json.load(db_file)
138        for entry in db_json:
139            if "file" not in entry:
140                continue
141            files.add(Path(entry["file"]))
142    return files
143
144
145def main():
146    parser = argparse.ArgumentParser(
147        description="Run clang-tidy against changed files, and "
148        "output diagnostics only for modified "
149        "lines."
150    )
151    parser.add_argument(
152        "-clang-tidy-binary",
153        metavar="PATH",
154        default="clang-tidy",
155        help="path to clang-tidy binary",
156    )
157    parser.add_argument(
158        "-p",
159        metavar="NUM",
160        default=0,
161        help="strip the smallest prefix containing P slashes",
162    )
163    parser.add_argument(
164        "-regex",
165        metavar="PATTERN",
166        default=None,
167        help="custom pattern selecting file paths to check "
168        "(case sensitive, overrides -iregex)",
169    )
170    parser.add_argument(
171        "-iregex",
172        metavar="PATTERN",
173        default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",
174        help="custom pattern selecting file paths to check "
175        "(case insensitive, overridden by -regex)",
176    )
177    parser.add_argument(
178        "-j",
179        type=int,
180        default=1,
181        help="number of tidy instances to be run in parallel.",
182    )
183    parser.add_argument(
184        "-timeout", type=int, default=None, help="timeout per each file in seconds."
185    )
186    parser.add_argument(
187        "-fix", action="store_true", default=False, help="apply suggested fixes"
188    )
189    parser.add_argument(
190        "-checks",
191        help="checks filter, when not specified, use clang-tidy " "default",
192        default="",
193    )
194    parser.add_argument(
195        "-config-file",
196        dest="config_file",
197        help="Specify the path of .clang-tidy or custom config file",
198        default="",
199    )
200    parser.add_argument("-use-color", action="store_true", help="Use colors in output")
201    parser.add_argument(
202        "-path", dest="build_path", help="Path used to read a compile command database."
203    )
204    if yaml:
205        parser.add_argument(
206            "-export-fixes",
207            metavar="FILE_OR_DIRECTORY",
208            dest="export_fixes",
209            help="A directory or a yaml file to store suggested fixes in, "
210            "which can be applied with clang-apply-replacements. If the "
211            "parameter is a directory, the fixes of each compilation unit are "
212            "stored in individual yaml files in the directory.",
213        )
214    else:
215        parser.add_argument(
216            "-export-fixes",
217            metavar="DIRECTORY",
218            dest="export_fixes",
219            help="A directory to store suggested fixes in, which can be applied "
220            "with clang-apply-replacements. The fixes of each compilation unit are "
221            "stored in individual yaml files in the directory.",
222        )
223    parser.add_argument(
224        "-extra-arg",
225        dest="extra_arg",
226        action="append",
227        default=[],
228        help="Additional argument to append to the compiler " "command line.",
229    )
230    parser.add_argument(
231        "-extra-arg-before",
232        dest="extra_arg_before",
233        action="append",
234        default=[],
235        help="Additional argument to prepend to the compiler " "command line.",
236    )
237    parser.add_argument(
238        "-quiet",
239        action="store_true",
240        default=False,
241        help="Run clang-tidy in quiet mode",
242    )
243    parser.add_argument(
244        "-load",
245        dest="plugins",
246        action="append",
247        default=[],
248        help="Load the specified plugin in clang-tidy.",
249    )
250    parser.add_argument(
251        "-allow-no-checks",
252        action="store_true",
253        help="Allow empty enabled checks.",
254    )
255    parser.add_argument(
256        "-only-check-in-db",
257        dest="skip_non_compiling",
258        default=False,
259        action="store_true",
260        help="Only check files in the compilation database",
261    )
262
263    clang_tidy_args = []
264    argv = sys.argv[1:]
265    if "--" in argv:
266        clang_tidy_args.extend(argv[argv.index("--") :])
267        argv = argv[: argv.index("--")]
268
269    args = parser.parse_args(argv)
270
271    compiling_files = get_compiling_files(args) if args.skip_non_compiling else None
272
273    # Extract changed lines for each file.
274    filename = None
275    lines_by_file = {}
276    for line in sys.stdin:
277        match = re.search(r'^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)
278        if match:
279            filename = match.group(2)
280        if filename is None:
281            continue
282
283        if args.regex is not None:
284            if not re.match("^%s$" % args.regex, filename):
285                continue
286        else:
287            if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
288                continue
289
290        # Skip any files not in the compiling list
291        if (
292            compiling_files is not None
293            and (Path.cwd() / filename) not in compiling_files
294        ):
295            continue
296
297        match = re.search(r"^@@.*\+(\d+)(,(\d+))?", line)
298        if match:
299            start_line = int(match.group(1))
300            line_count = 1
301            if match.group(3):
302                line_count = int(match.group(3))
303            if line_count == 0:
304                continue
305            end_line = start_line + line_count - 1
306            lines_by_file.setdefault(filename, []).append([start_line, end_line])
307
308    if not any(lines_by_file):
309        print("No relevant changes found.")
310        sys.exit(0)
311
312    max_task_count = args.j
313    if max_task_count == 0:
314        max_task_count = multiprocessing.cpu_count()
315    max_task_count = min(len(lines_by_file), max_task_count)
316
317    combine_fixes = False
318    export_fixes_dir = None
319    delete_fixes_dir = False
320    if args.export_fixes is not None:
321        # if a directory is given, create it if it does not exist
322        if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
323            args.export_fixes
324        ):
325            os.makedirs(args.export_fixes)
326
327        if not os.path.isdir(args.export_fixes):
328            if not yaml:
329                raise RuntimeError(
330                    "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
331                )
332
333            combine_fixes = True
334
335        if os.path.isdir(args.export_fixes):
336            export_fixes_dir = args.export_fixes
337
338    if combine_fixes:
339        export_fixes_dir = tempfile.mkdtemp()
340        delete_fixes_dir = True
341
342    # Tasks for clang-tidy.
343    task_queue = queue.Queue(max_task_count)
344    # A lock for console output.
345    lock = threading.Lock()
346
347    # List of files with a non-zero return code.
348    failed_files = []
349
350    # Run a pool of clang-tidy workers.
351    start_workers(
352        max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files)
353    )
354
355    # Form the common args list.
356    common_clang_tidy_args = []
357    if args.fix:
358        common_clang_tidy_args.append("-fix")
359    if args.checks != "":
360        common_clang_tidy_args.append("-checks=" + args.checks)
361    if args.config_file != "":
362        common_clang_tidy_args.append("-config-file=" + args.config_file)
363    if args.quiet:
364        common_clang_tidy_args.append("-quiet")
365    if args.build_path is not None:
366        common_clang_tidy_args.append("-p=%s" % args.build_path)
367    if args.use_color:
368        common_clang_tidy_args.append("--use-color")
369    if args.allow_no_checks:
370        common_clang_tidy_args.append("--allow-no-checks")
371    for arg in args.extra_arg:
372        common_clang_tidy_args.append("-extra-arg=%s" % arg)
373    for arg in args.extra_arg_before:
374        common_clang_tidy_args.append("-extra-arg-before=%s" % arg)
375    for plugin in args.plugins:
376        common_clang_tidy_args.append("-load=%s" % plugin)
377
378    for name in lines_by_file:
379        line_filter_json = json.dumps(
380            [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":")
381        )
382
383        # Run clang-tidy on files containing changes.
384        command = [args.clang_tidy_binary]
385        command.append("-line-filter=" + line_filter_json)
386        if args.export_fixes is not None:
387            # Get a temporary file. We immediately close the handle so clang-tidy can
388            # overwrite it.
389            (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir)
390            os.close(handle)
391            command.append("-export-fixes=" + tmp_name)
392        command.extend(common_clang_tidy_args)
393        command.append(name)
394        command.extend(clang_tidy_args)
395
396        task_queue.put(command)
397
398    # Application return code
399    return_code = 0
400
401    # Wait for all threads to be done.
402    task_queue.join()
403    # Application return code
404    return_code = 0
405    if failed_files:
406        return_code = 1
407
408    if combine_fixes:
409        print("Writing fixes to " + args.export_fixes + " ...")
410        try:
411            merge_replacement_files(export_fixes_dir, args.export_fixes)
412        except:
413            sys.stderr.write("Error exporting fixes.\n")
414            traceback.print_exc()
415            return_code = 1
416
417    if delete_fixes_dir:
418        shutil.rmtree(export_fixes_dir)
419    sys.exit(return_code)
420
421
422if __name__ == "__main__":
423    main()
424