1#!/usr/bin/env python3 2# 3# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===# 4# 5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 6# See https://llvm.org/LICENSE.txt for license information. 7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 8# 9# ===-----------------------------------------------------------------------===# 10 11r""" 12ClangTidy Diff Checker 13====================== 14 15This script reads input from a unified diff, runs clang-tidy on all changed 16files and outputs clang-tidy warnings in changed lines only. This is useful to 17detect clang-tidy regressions in the lines touched by a specific patch. 18Example usage for git/svn users: 19 20 git diff -U0 HEAD^ | clang-tidy-diff.py -p1 21 svn diff --diff-cmd=diff -x-U0 | \ 22 clang-tidy-diff.py -fix -checks=-*,modernize-use-override 23 24""" 25 26import argparse 27import glob 28import json 29import multiprocessing 30import os 31import re 32import shutil 33import subprocess 34import sys 35import tempfile 36import threading 37import traceback 38 39try: 40 import yaml 41except ImportError: 42 yaml = None 43 44is_py2 = sys.version[0] == "2" 45 46if is_py2: 47 import Queue as queue 48else: 49 import queue as queue 50 51 52def run_tidy(task_queue, lock, timeout, failed_files): 53 watchdog = None 54 while True: 55 command = task_queue.get() 56 try: 57 proc = subprocess.Popen( 58 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE 59 ) 60 61 if timeout is not None: 62 watchdog = threading.Timer(timeout, proc.kill) 63 watchdog.start() 64 65 stdout, stderr = proc.communicate() 66 if proc.returncode != 0: 67 if proc.returncode < 0: 68 msg = "Terminated by signal %d : %s\n" % ( 69 -proc.returncode, 70 " ".join(command), 71 ) 72 stderr += msg.encode("utf-8") 73 failed_files.append(command) 74 75 with lock: 76 sys.stdout.write(stdout.decode("utf-8") + "\n") 77 sys.stdout.flush() 78 if stderr: 79 sys.stderr.write(stderr.decode("utf-8") + "\n") 80 sys.stderr.flush() 81 except Exception as e: 82 with lock: 83 sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n") 84 finally: 85 with lock: 86 if not (timeout is None or watchdog is None): 87 if not watchdog.is_alive(): 88 sys.stderr.write( 89 "Terminated by timeout: " + " ".join(command) + "\n" 90 ) 91 watchdog.cancel() 92 task_queue.task_done() 93 94 95def start_workers(max_tasks, tidy_caller, arguments): 96 for _ in range(max_tasks): 97 t = threading.Thread(target=tidy_caller, args=arguments) 98 t.daemon = True 99 t.start() 100 101 102def merge_replacement_files(tmpdir, mergefile): 103 """Merge all replacement files in a directory into a single file""" 104 # The fixes suggested by clang-tidy >= 4.0.0 are given under 105 # the top level key 'Diagnostics' in the output yaml files 106 mergekey = "Diagnostics" 107 merged = [] 108 for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): 109 content = yaml.safe_load(open(replacefile, "r")) 110 if not content: 111 continue # Skip empty files. 112 merged.extend(content.get(mergekey, [])) 113 114 if merged: 115 # MainSourceFile: The key is required by the definition inside 116 # include/clang/Tooling/ReplacementsYaml.h, but the value 117 # is actually never used inside clang-apply-replacements, 118 # so we set it to '' here. 119 output = {"MainSourceFile": "", mergekey: merged} 120 with open(mergefile, "w") as out: 121 yaml.safe_dump(output, out) 122 else: 123 # Empty the file: 124 open(mergefile, "w").close() 125 126 127def main(): 128 parser = argparse.ArgumentParser( 129 description="Run clang-tidy against changed files, and " 130 "output diagnostics only for modified " 131 "lines." 132 ) 133 parser.add_argument( 134 "-clang-tidy-binary", 135 metavar="PATH", 136 default="clang-tidy", 137 help="path to clang-tidy binary", 138 ) 139 parser.add_argument( 140 "-p", 141 metavar="NUM", 142 default=0, 143 help="strip the smallest prefix containing P slashes", 144 ) 145 parser.add_argument( 146 "-regex", 147 metavar="PATTERN", 148 default=None, 149 help="custom pattern selecting file paths to check " 150 "(case sensitive, overrides -iregex)", 151 ) 152 parser.add_argument( 153 "-iregex", 154 metavar="PATTERN", 155 default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)", 156 help="custom pattern selecting file paths to check " 157 "(case insensitive, overridden by -regex)", 158 ) 159 parser.add_argument( 160 "-j", 161 type=int, 162 default=1, 163 help="number of tidy instances to be run in parallel.", 164 ) 165 parser.add_argument( 166 "-timeout", type=int, default=None, help="timeout per each file in seconds." 167 ) 168 parser.add_argument( 169 "-fix", action="store_true", default=False, help="apply suggested fixes" 170 ) 171 parser.add_argument( 172 "-checks", 173 help="checks filter, when not specified, use clang-tidy " "default", 174 default="", 175 ) 176 parser.add_argument("-use-color", action="store_true", help="Use colors in output") 177 parser.add_argument( 178 "-path", dest="build_path", help="Path used to read a compile command database." 179 ) 180 if yaml: 181 parser.add_argument( 182 "-export-fixes", 183 metavar="FILE_OR_DIRECTORY", 184 dest="export_fixes", 185 help="A directory or a yaml file to store suggested fixes in, " 186 "which can be applied with clang-apply-replacements. If the " 187 "parameter is a directory, the fixes of each compilation unit are " 188 "stored in individual yaml files in the directory.", 189 ) 190 else: 191 parser.add_argument( 192 "-export-fixes", 193 metavar="DIRECTORY", 194 dest="export_fixes", 195 help="A directory to store suggested fixes in, which can be applied " 196 "with clang-apply-replacements. The fixes of each compilation unit are " 197 "stored in individual yaml files in the directory.", 198 ) 199 parser.add_argument( 200 "-extra-arg", 201 dest="extra_arg", 202 action="append", 203 default=[], 204 help="Additional argument to append to the compiler " "command line.", 205 ) 206 parser.add_argument( 207 "-extra-arg-before", 208 dest="extra_arg_before", 209 action="append", 210 default=[], 211 help="Additional argument to prepend to the compiler " "command line.", 212 ) 213 parser.add_argument( 214 "-quiet", 215 action="store_true", 216 default=False, 217 help="Run clang-tidy in quiet mode", 218 ) 219 parser.add_argument( 220 "-load", 221 dest="plugins", 222 action="append", 223 default=[], 224 help="Load the specified plugin in clang-tidy.", 225 ) 226 227 clang_tidy_args = [] 228 argv = sys.argv[1:] 229 if "--" in argv: 230 clang_tidy_args.extend(argv[argv.index("--") :]) 231 argv = argv[: argv.index("--")] 232 233 args = parser.parse_args(argv) 234 235 # Extract changed lines for each file. 236 filename = None 237 lines_by_file = {} 238 for line in sys.stdin: 239 match = re.search('^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line) 240 if match: 241 filename = match.group(2) 242 if filename is None: 243 continue 244 245 if args.regex is not None: 246 if not re.match("^%s$" % args.regex, filename): 247 continue 248 else: 249 if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): 250 continue 251 252 match = re.search("^@@.*\+(\d+)(,(\d+))?", line) 253 if match: 254 start_line = int(match.group(1)) 255 line_count = 1 256 if match.group(3): 257 line_count = int(match.group(3)) 258 if line_count == 0: 259 continue 260 end_line = start_line + line_count - 1 261 lines_by_file.setdefault(filename, []).append([start_line, end_line]) 262 263 if not any(lines_by_file): 264 print("No relevant changes found.") 265 sys.exit(0) 266 267 max_task_count = args.j 268 if max_task_count == 0: 269 max_task_count = multiprocessing.cpu_count() 270 max_task_count = min(len(lines_by_file), max_task_count) 271 272 combine_fixes = False 273 export_fixes_dir = None 274 delete_fixes_dir = False 275 if args.export_fixes is not None: 276 # if a directory is given, create it if it does not exist 277 if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( 278 args.export_fixes 279 ): 280 os.makedirs(args.export_fixes) 281 282 if not os.path.isdir(args.export_fixes): 283 if not yaml: 284 raise RuntimeError( 285 "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory." 286 ) 287 288 combine_fixes = True 289 290 if os.path.isdir(args.export_fixes): 291 export_fixes_dir = args.export_fixes 292 293 if combine_fixes: 294 export_fixes_dir = tempfile.mkdtemp() 295 delete_fixes_dir = True 296 297 # Tasks for clang-tidy. 298 task_queue = queue.Queue(max_task_count) 299 # A lock for console output. 300 lock = threading.Lock() 301 302 # List of files with a non-zero return code. 303 failed_files = [] 304 305 # Run a pool of clang-tidy workers. 306 start_workers( 307 max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files) 308 ) 309 310 # Form the common args list. 311 common_clang_tidy_args = [] 312 if args.fix: 313 common_clang_tidy_args.append("-fix") 314 if args.checks != "": 315 common_clang_tidy_args.append("-checks=" + args.checks) 316 if args.quiet: 317 common_clang_tidy_args.append("-quiet") 318 if args.build_path is not None: 319 common_clang_tidy_args.append("-p=%s" % args.build_path) 320 if args.use_color: 321 common_clang_tidy_args.append("--use-color") 322 for arg in args.extra_arg: 323 common_clang_tidy_args.append("-extra-arg=%s" % arg) 324 for arg in args.extra_arg_before: 325 common_clang_tidy_args.append("-extra-arg-before=%s" % arg) 326 for plugin in args.plugins: 327 common_clang_tidy_args.append("-load=%s" % plugin) 328 329 for name in lines_by_file: 330 line_filter_json = json.dumps( 331 [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":") 332 ) 333 334 # Run clang-tidy on files containing changes. 335 command = [args.clang_tidy_binary] 336 command.append("-line-filter=" + line_filter_json) 337 if args.export_fixes is not None: 338 # Get a temporary file. We immediately close the handle so clang-tidy can 339 # overwrite it. 340 (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir) 341 os.close(handle) 342 command.append("-export-fixes=" + tmp_name) 343 command.extend(common_clang_tidy_args) 344 command.append(name) 345 command.extend(clang_tidy_args) 346 347 task_queue.put(command) 348 349 # Application return code 350 return_code = 0 351 352 # Wait for all threads to be done. 353 task_queue.join() 354 # Application return code 355 return_code = 0 356 if failed_files: 357 return_code = 1 358 359 if combine_fixes: 360 print("Writing fixes to " + args.export_fixes + " ...") 361 try: 362 merge_replacement_files(export_fixes_dir, args.export_fixes) 363 except: 364 sys.stderr.write("Error exporting fixes.\n") 365 traceback.print_exc() 366 return_code = 1 367 368 if delete_fixes_dir: 369 shutil.rmtree(export_fixes_dir) 370 sys.exit(return_code) 371 372 373if __name__ == "__main__": 374 main() 375