1#!/usr/bin/env python3 2# 3# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===# 4# 5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 6# See https://llvm.org/LICENSE.txt for license information. 7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 8# 9# ===-----------------------------------------------------------------------===# 10 11r""" 12ClangTidy Diff Checker 13====================== 14 15This script reads input from a unified diff, runs clang-tidy on all changed 16files and outputs clang-tidy warnings in changed lines only. This is useful to 17detect clang-tidy regressions in the lines touched by a specific patch. 18Example usage for git/svn users: 19 20 git diff -U0 HEAD^ | clang-tidy-diff.py -p1 21 svn diff --diff-cmd=diff -x-U0 | \ 22 clang-tidy-diff.py -fix -checks=-*,modernize-use-override 23 24""" 25 26import argparse 27import glob 28import json 29import multiprocessing 30import os 31import re 32import shutil 33import subprocess 34import sys 35import tempfile 36import threading 37import traceback 38 39try: 40 import yaml 41except ImportError: 42 yaml = None 43 44is_py2 = sys.version[0] == "2" 45 46if is_py2: 47 import Queue as queue 48else: 49 import queue as queue 50 51 52def run_tidy(task_queue, lock, timeout, failed_files): 53 watchdog = None 54 while True: 55 command = task_queue.get() 56 try: 57 proc = subprocess.Popen( 58 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE 59 ) 60 61 if timeout is not None: 62 watchdog = threading.Timer(timeout, proc.kill) 63 watchdog.start() 64 65 stdout, stderr = proc.communicate() 66 if proc.returncode != 0: 67 if proc.returncode < 0: 68 msg = "Terminated by signal %d : %s\n" % ( 69 -proc.returncode, 70 " ".join(command), 71 ) 72 stderr += msg.encode("utf-8") 73 failed_files.append(command) 74 75 with lock: 76 sys.stdout.write(stdout.decode("utf-8") + "\n") 77 sys.stdout.flush() 78 if stderr: 79 sys.stderr.write(stderr.decode("utf-8") + "\n") 80 sys.stderr.flush() 81 except Exception as e: 82 with lock: 83 sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n") 84 finally: 85 with lock: 86 if not (timeout is None or watchdog is None): 87 if not watchdog.is_alive(): 88 sys.stderr.write( 89 "Terminated by timeout: " + " ".join(command) + "\n" 90 ) 91 watchdog.cancel() 92 task_queue.task_done() 93 94 95def start_workers(max_tasks, tidy_caller, arguments): 96 for _ in range(max_tasks): 97 t = threading.Thread(target=tidy_caller, args=arguments) 98 t.daemon = True 99 t.start() 100 101 102def merge_replacement_files(tmpdir, mergefile): 103 """Merge all replacement files in a directory into a single file""" 104 # The fixes suggested by clang-tidy >= 4.0.0 are given under 105 # the top level key 'Diagnostics' in the output yaml files 106 mergekey = "Diagnostics" 107 merged = [] 108 for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): 109 content = yaml.safe_load(open(replacefile, "r")) 110 if not content: 111 continue # Skip empty files. 112 merged.extend(content.get(mergekey, [])) 113 114 if merged: 115 # MainSourceFile: The key is required by the definition inside 116 # include/clang/Tooling/ReplacementsYaml.h, but the value 117 # is actually never used inside clang-apply-replacements, 118 # so we set it to '' here. 119 output = {"MainSourceFile": "", mergekey: merged} 120 with open(mergefile, "w") as out: 121 yaml.safe_dump(output, out) 122 else: 123 # Empty the file: 124 open(mergefile, "w").close() 125 126 127def main(): 128 parser = argparse.ArgumentParser( 129 description="Run clang-tidy against changed files, and " 130 "output diagnostics only for modified " 131 "lines." 132 ) 133 parser.add_argument( 134 "-clang-tidy-binary", 135 metavar="PATH", 136 default="clang-tidy", 137 help="path to clang-tidy binary", 138 ) 139 parser.add_argument( 140 "-p", 141 metavar="NUM", 142 default=0, 143 help="strip the smallest prefix containing P slashes", 144 ) 145 parser.add_argument( 146 "-regex", 147 metavar="PATTERN", 148 default=None, 149 help="custom pattern selecting file paths to check " 150 "(case sensitive, overrides -iregex)", 151 ) 152 parser.add_argument( 153 "-iregex", 154 metavar="PATTERN", 155 default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)", 156 help="custom pattern selecting file paths to check " 157 "(case insensitive, overridden by -regex)", 158 ) 159 parser.add_argument( 160 "-j", 161 type=int, 162 default=1, 163 help="number of tidy instances to be run in parallel.", 164 ) 165 parser.add_argument( 166 "-timeout", type=int, default=None, help="timeout per each file in seconds." 167 ) 168 parser.add_argument( 169 "-fix", action="store_true", default=False, help="apply suggested fixes" 170 ) 171 parser.add_argument( 172 "-checks", 173 help="checks filter, when not specified, use clang-tidy " "default", 174 default="", 175 ) 176 parser.add_argument("-use-color", action="store_true", help="Use colors in output") 177 parser.add_argument( 178 "-path", dest="build_path", help="Path used to read a compile command database." 179 ) 180 if yaml: 181 parser.add_argument( 182 "-export-fixes", 183 metavar="FILE_OR_DIRECTORY", 184 dest="export_fixes", 185 help="A directory or a yaml file to store suggested fixes in, " 186 "which can be applied with clang-apply-replacements. If the " 187 "parameter is a directory, the fixes of each compilation unit are " 188 "stored in individual yaml files in the directory.", 189 ) 190 parser.add_argument( 191 "-extra-arg", 192 dest="extra_arg", 193 action="append", 194 default=[], 195 help="Additional argument to append to the compiler " "command line.", 196 ) 197 parser.add_argument( 198 "-extra-arg-before", 199 dest="extra_arg_before", 200 action="append", 201 default=[], 202 help="Additional argument to prepend to the compiler " "command line.", 203 ) 204 parser.add_argument( 205 "-quiet", 206 action="store_true", 207 default=False, 208 help="Run clang-tidy in quiet mode", 209 ) 210 parser.add_argument( 211 "-load", 212 dest="plugins", 213 action="append", 214 default=[], 215 help="Load the specified plugin in clang-tidy.", 216 ) 217 218 clang_tidy_args = [] 219 argv = sys.argv[1:] 220 if "--" in argv: 221 clang_tidy_args.extend(argv[argv.index("--") :]) 222 argv = argv[: argv.index("--")] 223 224 args = parser.parse_args(argv) 225 226 # Extract changed lines for each file. 227 filename = None 228 lines_by_file = {} 229 for line in sys.stdin: 230 match = re.search('^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line) 231 if match: 232 filename = match.group(2) 233 if filename is None: 234 continue 235 236 if args.regex is not None: 237 if not re.match("^%s$" % args.regex, filename): 238 continue 239 else: 240 if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): 241 continue 242 243 match = re.search("^@@.*\+(\d+)(,(\d+))?", line) 244 if match: 245 start_line = int(match.group(1)) 246 line_count = 1 247 if match.group(3): 248 line_count = int(match.group(3)) 249 if line_count == 0: 250 continue 251 end_line = start_line + line_count - 1 252 lines_by_file.setdefault(filename, []).append([start_line, end_line]) 253 254 if not any(lines_by_file): 255 print("No relevant changes found.") 256 sys.exit(0) 257 258 max_task_count = args.j 259 if max_task_count == 0: 260 max_task_count = multiprocessing.cpu_count() 261 max_task_count = min(len(lines_by_file), max_task_count) 262 263 combine_fixes = False 264 export_fixes_dir = None 265 delete_fixes_dir = False 266 if args.export_fixes is not None: 267 # if a directory is given, create it if it does not exist 268 if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( 269 args.export_fixes 270 ): 271 os.makedirs(args.export_fixes) 272 273 if not os.path.isdir(args.export_fixes) and yaml: 274 combine_fixes = True 275 276 if os.path.isdir(args.export_fixes): 277 export_fixes_dir = args.export_fixes 278 279 if combine_fixes: 280 export_fixes_dir = tempfile.mkdtemp() 281 delete_fixes_dir = True 282 283 # Tasks for clang-tidy. 284 task_queue = queue.Queue(max_task_count) 285 # A lock for console output. 286 lock = threading.Lock() 287 288 # List of files with a non-zero return code. 289 failed_files = [] 290 291 # Run a pool of clang-tidy workers. 292 start_workers( 293 max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files) 294 ) 295 296 # Form the common args list. 297 common_clang_tidy_args = [] 298 if args.fix: 299 common_clang_tidy_args.append("-fix") 300 if args.checks != "": 301 common_clang_tidy_args.append("-checks=" + args.checks) 302 if args.quiet: 303 common_clang_tidy_args.append("-quiet") 304 if args.build_path is not None: 305 common_clang_tidy_args.append("-p=%s" % args.build_path) 306 if args.use_color: 307 common_clang_tidy_args.append("--use-color") 308 for arg in args.extra_arg: 309 common_clang_tidy_args.append("-extra-arg=%s" % arg) 310 for arg in args.extra_arg_before: 311 common_clang_tidy_args.append("-extra-arg-before=%s" % arg) 312 for plugin in args.plugins: 313 common_clang_tidy_args.append("-load=%s" % plugin) 314 315 for name in lines_by_file: 316 line_filter_json = json.dumps( 317 [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":") 318 ) 319 320 # Run clang-tidy on files containing changes. 321 command = [args.clang_tidy_binary] 322 command.append("-line-filter=" + line_filter_json) 323 if args.export_fixes is not None: 324 # Get a temporary file. We immediately close the handle so clang-tidy can 325 # overwrite it. 326 (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir) 327 os.close(handle) 328 command.append("-export-fixes=" + tmp_name) 329 command.extend(common_clang_tidy_args) 330 command.append(name) 331 command.extend(clang_tidy_args) 332 333 task_queue.put(command) 334 335 # Application return code 336 return_code = 0 337 338 # Wait for all threads to be done. 339 task_queue.join() 340 # Application return code 341 return_code = 0 342 if failed_files: 343 return_code = 1 344 345 if combine_fixes: 346 print("Writing fixes to " + args.export_fixes + " ...") 347 try: 348 merge_replacement_files(export_fixes_dir, args.export_fixes) 349 except: 350 sys.stderr.write("Error exporting fixes.\n") 351 traceback.print_exc() 352 return_code = 1 353 354 if delete_fixes_dir: 355 shutil.rmtree(export_fixes_dir) 356 sys.exit(return_code) 357 358 359if __name__ == "__main__": 360 main() 361