1#!/usr/bin/env python3 2# 3# ===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===# 4# 5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 6# See https://llvm.org/LICENSE.txt for license information. 7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 8# 9# ===-----------------------------------------------------------------------===# 10# FIXME: Integrate with clang-tidy-diff.py 11 12 13""" 14Parallel clang-tidy runner 15========================== 16 17Runs clang-tidy over all files in a compilation database. Requires clang-tidy 18and clang-apply-replacements in $PATH. 19 20Example invocations. 21- Run clang-tidy on all files in the current working directory with a default 22 set of checks and show warnings in the cpp files and all project headers. 23 run-clang-tidy.py $PWD 24 25- Fix all header guards. 26 run-clang-tidy.py -fix -checks=-*,llvm-header-guard 27 28- Fix all header guards included from clang-tidy and header guards 29 for clang-tidy headers. 30 run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ 31 -header-filter=extra/clang-tidy 32 33Compilation database setup: 34http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html 35""" 36 37import argparse 38import asyncio 39from dataclasses import dataclass 40import glob 41import json 42import multiprocessing 43import os 44import re 45import shutil 46import subprocess 47import sys 48import tempfile 49import time 50import traceback 51from types import ModuleType 52from typing import Any, Awaitable, Callable, List, Optional, TypeVar 53 54 55yaml: Optional[ModuleType] = None 56try: 57 import yaml 58except ImportError: 59 yaml = None 60 61 62def strtobool(val: str) -> bool: 63 """Convert a string representation of truth to a bool following LLVM's CLI argument parsing.""" 64 65 val = val.lower() 66 if val in ["", "true", "1"]: 67 return True 68 elif val in ["false", "0"]: 69 return False 70 71 # Return ArgumentTypeError so that argparse does not substitute its own error message 72 raise argparse.ArgumentTypeError( 73 f"'{val}' is invalid value for boolean argument! Try 0 or 1." 74 ) 75 76 77def find_compilation_database(path: str) -> str: 78 """Adjusts the directory until a compilation database is found.""" 79 result = os.path.realpath("./") 80 while not os.path.isfile(os.path.join(result, path)): 81 parent = os.path.dirname(result) 82 if result == parent: 83 print("Error: could not find compilation database.") 84 sys.exit(1) 85 result = parent 86 return result 87 88 89def get_tidy_invocation( 90 f: str, 91 clang_tidy_binary: str, 92 checks: str, 93 tmpdir: Optional[str], 94 build_path: str, 95 header_filter: Optional[str], 96 allow_enabling_alpha_checkers: bool, 97 extra_arg: List[str], 98 extra_arg_before: List[str], 99 quiet: bool, 100 config_file_path: str, 101 config: str, 102 line_filter: Optional[str], 103 use_color: bool, 104 plugins: List[str], 105 warnings_as_errors: Optional[str], 106 exclude_header_filter: Optional[str], 107 allow_no_checks: bool, 108) -> List[str]: 109 """Gets a command line for clang-tidy.""" 110 start = [clang_tidy_binary] 111 if allow_enabling_alpha_checkers: 112 start.append("-allow-enabling-analyzer-alpha-checkers") 113 if exclude_header_filter is not None: 114 start.append(f"--exclude-header-filter={exclude_header_filter}") 115 if header_filter is not None: 116 start.append(f"-header-filter={header_filter}") 117 if line_filter is not None: 118 start.append(f"-line-filter={line_filter}") 119 if use_color is not None: 120 if use_color: 121 start.append("--use-color") 122 else: 123 start.append("--use-color=false") 124 if checks: 125 start.append(f"-checks={checks}") 126 if tmpdir is not None: 127 start.append("-export-fixes") 128 # Get a temporary file. We immediately close the handle so clang-tidy can 129 # overwrite it. 130 (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) 131 os.close(handle) 132 start.append(name) 133 for arg in extra_arg: 134 start.append(f"-extra-arg={arg}") 135 for arg in extra_arg_before: 136 start.append(f"-extra-arg-before={arg}") 137 start.append(f"-p={build_path}") 138 if quiet: 139 start.append("-quiet") 140 if config_file_path: 141 start.append(f"--config-file={config_file_path}") 142 elif config: 143 start.append(f"-config={config}") 144 for plugin in plugins: 145 start.append(f"-load={plugin}") 146 if warnings_as_errors: 147 start.append(f"--warnings-as-errors={warnings_as_errors}") 148 if allow_no_checks: 149 start.append("--allow-no-checks") 150 start.append(f) 151 return start 152 153 154def merge_replacement_files(tmpdir: str, mergefile: str) -> None: 155 """Merge all replacement files in a directory into a single file""" 156 assert yaml 157 # The fixes suggested by clang-tidy >= 4.0.0 are given under 158 # the top level key 'Diagnostics' in the output yaml files 159 mergekey = "Diagnostics" 160 merged = [] 161 for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): 162 content = yaml.safe_load(open(replacefile, "r")) 163 if not content: 164 continue # Skip empty files. 165 merged.extend(content.get(mergekey, [])) 166 167 if merged: 168 # MainSourceFile: The key is required by the definition inside 169 # include/clang/Tooling/ReplacementsYaml.h, but the value 170 # is actually never used inside clang-apply-replacements, 171 # so we set it to '' here. 172 output = {"MainSourceFile": "", mergekey: merged} 173 with open(mergefile, "w") as out: 174 yaml.safe_dump(output, out) 175 else: 176 # Empty the file: 177 open(mergefile, "w").close() 178 179 180def find_binary(arg: str, name: str, build_path: str) -> str: 181 """Get the path for a binary or exit""" 182 if arg: 183 if shutil.which(arg): 184 return arg 185 else: 186 raise SystemExit( 187 f"error: passed binary '{arg}' was not found or is not executable" 188 ) 189 190 built_path = os.path.join(build_path, "bin", name) 191 binary = shutil.which(name) or shutil.which(built_path) 192 if binary: 193 return binary 194 else: 195 raise SystemExit(f"error: failed to find {name} in $PATH or at {built_path}") 196 197 198def apply_fixes( 199 args: argparse.Namespace, clang_apply_replacements_binary: str, tmpdir: str 200) -> None: 201 """Calls clang-apply-fixes on a given directory.""" 202 invocation = [clang_apply_replacements_binary] 203 invocation.append("-ignore-insert-conflict") 204 if args.format: 205 invocation.append("-format") 206 if args.style: 207 invocation.append(f"-style={args.style}") 208 invocation.append(tmpdir) 209 subprocess.call(invocation) 210 211 212# FIXME Python 3.12: This can be simplified out with run_with_semaphore[T](...). 213T = TypeVar("T") 214 215 216async def run_with_semaphore( 217 semaphore: asyncio.Semaphore, 218 f: Callable[..., Awaitable[T]], 219 *args: Any, 220 **kwargs: Any, 221) -> T: 222 async with semaphore: 223 return await f(*args, **kwargs) 224 225 226@dataclass 227class ClangTidyResult: 228 filename: str 229 invocation: List[str] 230 returncode: int 231 stdout: str 232 stderr: str 233 elapsed: float 234 235 236async def run_tidy( 237 args: argparse.Namespace, 238 name: str, 239 clang_tidy_binary: str, 240 tmpdir: str, 241 build_path: str, 242) -> ClangTidyResult: 243 """ 244 Runs clang-tidy on a single file and returns the result. 245 """ 246 invocation = get_tidy_invocation( 247 name, 248 clang_tidy_binary, 249 args.checks, 250 tmpdir, 251 build_path, 252 args.header_filter, 253 args.allow_enabling_alpha_checkers, 254 args.extra_arg, 255 args.extra_arg_before, 256 args.quiet, 257 args.config_file, 258 args.config, 259 args.line_filter, 260 args.use_color, 261 args.plugins, 262 args.warnings_as_errors, 263 args.exclude_header_filter, 264 args.allow_no_checks, 265 ) 266 267 try: 268 process = await asyncio.create_subprocess_exec( 269 *invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE 270 ) 271 start = time.time() 272 stdout, stderr = await process.communicate() 273 end = time.time() 274 except asyncio.CancelledError: 275 process.terminate() 276 await process.wait() 277 raise 278 279 assert process.returncode is not None 280 return ClangTidyResult( 281 name, 282 invocation, 283 process.returncode, 284 stdout.decode("UTF-8"), 285 stderr.decode("UTF-8"), 286 end - start, 287 ) 288 289 290async def main() -> None: 291 parser = argparse.ArgumentParser( 292 description="Runs clang-tidy over all files " 293 "in a compilation database. Requires " 294 "clang-tidy and clang-apply-replacements in " 295 "$PATH or in your build directory." 296 ) 297 parser.add_argument( 298 "-allow-enabling-alpha-checkers", 299 action="store_true", 300 help="Allow alpha checkers from clang-analyzer.", 301 ) 302 parser.add_argument( 303 "-clang-tidy-binary", metavar="PATH", help="Path to clang-tidy binary." 304 ) 305 parser.add_argument( 306 "-clang-apply-replacements-binary", 307 metavar="PATH", 308 help="Path to clang-apply-replacements binary.", 309 ) 310 parser.add_argument( 311 "-checks", 312 default=None, 313 help="Checks filter, when not specified, use clang-tidy default.", 314 ) 315 config_group = parser.add_mutually_exclusive_group() 316 config_group.add_argument( 317 "-config", 318 default=None, 319 help="Specifies a configuration in YAML/JSON format: " 320 " -config=\"{Checks: '*', " 321 ' CheckOptions: {x: y}}" ' 322 "When the value is empty, clang-tidy will " 323 "attempt to find a file named .clang-tidy for " 324 "each source file in its parent directories.", 325 ) 326 config_group.add_argument( 327 "-config-file", 328 default=None, 329 help="Specify the path of .clang-tidy or custom config " 330 "file: e.g. -config-file=/some/path/myTidyConfigFile. " 331 "This option internally works exactly the same way as " 332 "-config option after reading specified config file. " 333 "Use either -config-file or -config, not both.", 334 ) 335 parser.add_argument( 336 "-exclude-header-filter", 337 default=None, 338 help="Regular expression matching the names of the " 339 "headers to exclude diagnostics from. Diagnostics from " 340 "the main file of each translation unit are always " 341 "displayed.", 342 ) 343 parser.add_argument( 344 "-header-filter", 345 default=None, 346 help="Regular expression matching the names of the " 347 "headers to output diagnostics from. Diagnostics from " 348 "the main file of each translation unit are always " 349 "displayed.", 350 ) 351 parser.add_argument( 352 "-source-filter", 353 default=None, 354 help="Regular expression matching the names of the " 355 "source files from compilation database to output " 356 "diagnostics from.", 357 ) 358 parser.add_argument( 359 "-line-filter", 360 default=None, 361 help="List of files with line ranges to filter the warnings.", 362 ) 363 if yaml: 364 parser.add_argument( 365 "-export-fixes", 366 metavar="file_or_directory", 367 dest="export_fixes", 368 help="A directory or a yaml file to store suggested fixes in, " 369 "which can be applied with clang-apply-replacements. If the " 370 "parameter is a directory, the fixes of each compilation unit are " 371 "stored in individual yaml files in the directory.", 372 ) 373 else: 374 parser.add_argument( 375 "-export-fixes", 376 metavar="directory", 377 dest="export_fixes", 378 help="A directory to store suggested fixes in, which can be applied " 379 "with clang-apply-replacements. The fixes of each compilation unit are " 380 "stored in individual yaml files in the directory.", 381 ) 382 parser.add_argument( 383 "-j", 384 type=int, 385 default=0, 386 help="Number of tidy instances to be run in parallel.", 387 ) 388 parser.add_argument( 389 "files", 390 nargs="*", 391 default=[".*"], 392 help="Files to be processed (regex on path).", 393 ) 394 parser.add_argument("-fix", action="store_true", help="apply fix-its.") 395 parser.add_argument( 396 "-format", action="store_true", help="Reformat code after applying fixes." 397 ) 398 parser.add_argument( 399 "-style", 400 default="file", 401 help="The style of reformat code after applying fixes.", 402 ) 403 parser.add_argument( 404 "-use-color", 405 type=strtobool, 406 nargs="?", 407 const=True, 408 help="Use colors in diagnostics, overriding clang-tidy's" 409 " default behavior. This option overrides the 'UseColor" 410 "' option in .clang-tidy file, if any.", 411 ) 412 parser.add_argument( 413 "-p", dest="build_path", help="Path used to read a compile command database." 414 ) 415 parser.add_argument( 416 "-extra-arg", 417 dest="extra_arg", 418 action="append", 419 default=[], 420 help="Additional argument to append to the compiler command line.", 421 ) 422 parser.add_argument( 423 "-extra-arg-before", 424 dest="extra_arg_before", 425 action="append", 426 default=[], 427 help="Additional argument to prepend to the compiler command line.", 428 ) 429 parser.add_argument( 430 "-quiet", action="store_true", help="Run clang-tidy in quiet mode." 431 ) 432 parser.add_argument( 433 "-load", 434 dest="plugins", 435 action="append", 436 default=[], 437 help="Load the specified plugin in clang-tidy.", 438 ) 439 parser.add_argument( 440 "-warnings-as-errors", 441 default=None, 442 help="Upgrades warnings to errors. Same format as '-checks'.", 443 ) 444 parser.add_argument( 445 "-allow-no-checks", 446 action="store_true", 447 help="Allow empty enabled checks.", 448 ) 449 args = parser.parse_args() 450 451 db_path = "compile_commands.json" 452 453 if args.build_path is not None: 454 build_path = args.build_path 455 else: 456 # Find our database 457 build_path = find_compilation_database(db_path) 458 459 clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy", build_path) 460 461 if args.fix: 462 clang_apply_replacements_binary = find_binary( 463 args.clang_apply_replacements_binary, "clang-apply-replacements", build_path 464 ) 465 466 combine_fixes = False 467 export_fixes_dir: Optional[str] = None 468 delete_fixes_dir = False 469 if args.export_fixes is not None: 470 # if a directory is given, create it if it does not exist 471 if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( 472 args.export_fixes 473 ): 474 os.makedirs(args.export_fixes) 475 476 if not os.path.isdir(args.export_fixes): 477 if not yaml: 478 raise RuntimeError( 479 "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory." 480 ) 481 482 combine_fixes = True 483 484 if os.path.isdir(args.export_fixes): 485 export_fixes_dir = args.export_fixes 486 487 if export_fixes_dir is None and (args.fix or combine_fixes): 488 export_fixes_dir = tempfile.mkdtemp() 489 delete_fixes_dir = True 490 491 try: 492 invocation = get_tidy_invocation( 493 "", 494 clang_tidy_binary, 495 args.checks, 496 None, 497 build_path, 498 args.header_filter, 499 args.allow_enabling_alpha_checkers, 500 args.extra_arg, 501 args.extra_arg_before, 502 args.quiet, 503 args.config_file, 504 args.config, 505 args.line_filter, 506 args.use_color, 507 args.plugins, 508 args.warnings_as_errors, 509 args.exclude_header_filter, 510 args.allow_no_checks, 511 ) 512 invocation.append("-list-checks") 513 invocation.append("-") 514 # Even with -quiet we still want to check if we can call clang-tidy. 515 subprocess.check_call( 516 invocation, stdout=subprocess.DEVNULL if args.quiet else None 517 ) 518 except: 519 print("Unable to run clang-tidy.", file=sys.stderr) 520 sys.exit(1) 521 522 # Load the database and extract all files. 523 with open(os.path.join(build_path, db_path)) as f: 524 database = json.load(f) 525 files = {os.path.abspath(os.path.join(e["directory"], e["file"])) for e in database} 526 number_files_in_database = len(files) 527 528 # Filter source files from compilation database. 529 if args.source_filter: 530 try: 531 source_filter_re = re.compile(args.source_filter) 532 except: 533 print( 534 "Error: unable to compile regex from arg -source-filter:", 535 file=sys.stderr, 536 ) 537 traceback.print_exc() 538 sys.exit(1) 539 files = {f for f in files if source_filter_re.match(f)} 540 541 max_task = args.j 542 if max_task == 0: 543 max_task = multiprocessing.cpu_count() 544 545 # Build up a big regexy filter from all command line arguments. 546 file_name_re = re.compile("|".join(args.files)) 547 files = {f for f in files if file_name_re.search(f)} 548 549 print( 550 "Running clang-tidy for", 551 len(files), 552 "files out of", 553 number_files_in_database, 554 "in compilation database ...", 555 ) 556 557 returncode = 0 558 semaphore = asyncio.Semaphore(max_task) 559 tasks = [ 560 asyncio.create_task( 561 run_with_semaphore( 562 semaphore, 563 run_tidy, 564 args, 565 f, 566 clang_tidy_binary, 567 export_fixes_dir, 568 build_path, 569 ) 570 ) 571 for f in files 572 ] 573 574 try: 575 for i, coro in enumerate(asyncio.as_completed(tasks)): 576 result = await coro 577 if result.returncode != 0: 578 returncode = 1 579 if result.returncode < 0: 580 result.stderr += f"{result.filename}: terminated by signal {-result.returncode}\n" 581 progress = f"[{i + 1: >{len(f'{len(files)}')}}/{len(files)}]" 582 runtime = f"[{result.elapsed:.1f}s]" 583 print(f"{progress}{runtime} {' '.join(result.invocation)}") 584 if result.stdout: 585 print(result.stdout, end=("" if result.stderr else "\n")) 586 if result.stderr: 587 print(result.stderr) 588 except asyncio.CancelledError: 589 print("\nCtrl-C detected, goodbye.") 590 for task in tasks: 591 task.cancel() 592 if delete_fixes_dir: 593 assert export_fixes_dir 594 shutil.rmtree(export_fixes_dir) 595 return 596 597 if combine_fixes: 598 print(f"Writing fixes to {args.export_fixes} ...") 599 try: 600 assert export_fixes_dir 601 merge_replacement_files(export_fixes_dir, args.export_fixes) 602 except: 603 print("Error exporting fixes.\n", file=sys.stderr) 604 traceback.print_exc() 605 returncode = 1 606 607 if args.fix: 608 print("Applying fixes ...") 609 try: 610 assert export_fixes_dir 611 apply_fixes(args, clang_apply_replacements_binary, export_fixes_dir) 612 except: 613 print("Error applying fixes.\n", file=sys.stderr) 614 traceback.print_exc() 615 returncode = 1 616 617 if delete_fixes_dir: 618 assert export_fixes_dir 619 shutil.rmtree(export_fixes_dir) 620 sys.exit(returncode) 621 622 623if __name__ == "__main__": 624 try: 625 asyncio.run(main()) 626 except KeyboardInterrupt: 627 pass 628