1 //===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Frontend/CompilerInstance.h" 10 #include "clang/Tooling/CommonOptionsParser.h" 11 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" 12 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" 13 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" 14 #include "clang/Tooling/JSONCompilationDatabase.h" 15 #include "llvm/Support/CommandLine.h" 16 #include "llvm/Support/FileUtilities.h" 17 #include "llvm/Support/InitLLVM.h" 18 #include "llvm/Support/JSON.h" 19 #include "llvm/Support/Program.h" 20 #include "llvm/Support/Signals.h" 21 #include "llvm/Support/ThreadPool.h" 22 #include "llvm/Support/Threading.h" 23 #include <mutex> 24 #include <thread> 25 26 using namespace clang; 27 using namespace tooling::dependencies; 28 29 namespace { 30 31 class SharedStream { 32 public: 33 SharedStream(raw_ostream &OS) : OS(OS) {} 34 void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) { 35 std::unique_lock<std::mutex> LockGuard(Lock); 36 Fn(OS); 37 OS.flush(); 38 } 39 40 private: 41 std::mutex Lock; 42 raw_ostream &OS; 43 }; 44 45 class ResourceDirectoryCache { 46 public: 47 /// findResourceDir finds the resource directory relative to the clang 48 /// compiler being used in Args, by running it with "-print-resource-dir" 49 /// option and cache the results for reuse. \returns resource directory path 50 /// associated with the given invocation command or empty string if the 51 /// compiler path is NOT an absolute path. 52 StringRef findResourceDir(const tooling::CommandLineArguments &Args) { 53 if (Args.size() < 1) 54 return ""; 55 56 const std::string &ClangBinaryPath = Args[0]; 57 if (!llvm::sys::path::is_absolute(ClangBinaryPath)) 58 return ""; 59 60 const std::string &ClangBinaryName = 61 std::string(llvm::sys::path::filename(ClangBinaryPath)); 62 63 std::unique_lock<std::mutex> LockGuard(CacheLock); 64 const auto &CachedResourceDir = Cache.find(ClangBinaryPath); 65 if (CachedResourceDir != Cache.end()) 66 return CachedResourceDir->second; 67 68 std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName, 69 "-print-resource-dir"}; 70 llvm::SmallString<64> OutputFile, ErrorFile; 71 llvm::sys::fs::createTemporaryFile("print-resource-dir-output", 72 "" /*no-suffix*/, OutputFile); 73 llvm::sys::fs::createTemporaryFile("print-resource-dir-error", 74 "" /*no-suffix*/, ErrorFile); 75 llvm::FileRemover OutputRemover(OutputFile.c_str()); 76 llvm::FileRemover ErrorRemover(ErrorFile.c_str()); 77 llvm::Optional<StringRef> Redirects[] = { 78 {""}, // Stdin 79 StringRef(OutputFile), 80 StringRef(ErrorFile), 81 }; 82 if (const int RC = llvm::sys::ExecuteAndWait( 83 ClangBinaryPath, PrintResourceDirArgs, {}, Redirects)) { 84 auto ErrorBuf = llvm::MemoryBuffer::getFile(ErrorFile.c_str()); 85 llvm::errs() << ErrorBuf.get()->getBuffer(); 86 return ""; 87 } 88 89 auto OutputBuf = llvm::MemoryBuffer::getFile(OutputFile.c_str()); 90 if (!OutputBuf) 91 return ""; 92 StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n'); 93 94 Cache[ClangBinaryPath] = Output.str(); 95 return Cache[ClangBinaryPath]; 96 } 97 98 private: 99 std::map<std::string, std::string> Cache; 100 std::mutex CacheLock; 101 }; 102 103 llvm::cl::opt<bool> Help("h", llvm::cl::desc("Alias for -help"), 104 llvm::cl::Hidden); 105 106 llvm::cl::OptionCategory DependencyScannerCategory("Tool options"); 107 108 static llvm::cl::opt<ScanningMode> ScanMode( 109 "mode", 110 llvm::cl::desc("The preprocessing mode used to compute the dependencies"), 111 llvm::cl::values( 112 clEnumValN(ScanningMode::MinimizedSourcePreprocessing, 113 "preprocess-minimized-sources", 114 "The set of dependencies is computed by preprocessing the " 115 "source files that were minimized to only include the " 116 "contents that might affect the dependencies"), 117 clEnumValN(ScanningMode::CanonicalPreprocessing, "preprocess", 118 "The set of dependencies is computed by preprocessing the " 119 "unmodified source files")), 120 llvm::cl::init(ScanningMode::MinimizedSourcePreprocessing), 121 llvm::cl::cat(DependencyScannerCategory)); 122 123 static llvm::cl::opt<ScanningOutputFormat> Format( 124 "format", llvm::cl::desc("The output format for the dependencies"), 125 llvm::cl::values(clEnumValN(ScanningOutputFormat::Make, "make", 126 "Makefile compatible dep file"), 127 clEnumValN(ScanningOutputFormat::Full, "experimental-full", 128 "Full dependency graph suitable" 129 " for explicitly building modules. This format " 130 "is experimental and will change.")), 131 llvm::cl::init(ScanningOutputFormat::Make), 132 llvm::cl::cat(DependencyScannerCategory)); 133 134 static llvm::cl::opt<bool> FullCommandLine( 135 "full-command-line", 136 llvm::cl::desc("Include the full command lines to use to build modules"), 137 llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory)); 138 139 llvm::cl::opt<unsigned> 140 NumThreads("j", llvm::cl::Optional, 141 llvm::cl::desc("Number of worker threads to use (default: use " 142 "all concurrent threads)"), 143 llvm::cl::init(0), llvm::cl::cat(DependencyScannerCategory)); 144 145 llvm::cl::opt<std::string> 146 CompilationDB("compilation-database", 147 llvm::cl::desc("Compilation database"), llvm::cl::Required, 148 llvm::cl::cat(DependencyScannerCategory)); 149 150 llvm::cl::opt<bool> ReuseFileManager( 151 "reuse-filemanager", 152 llvm::cl::desc("Reuse the file manager and its cache between invocations."), 153 llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory)); 154 155 llvm::cl::opt<bool> SkipExcludedPPRanges( 156 "skip-excluded-pp-ranges", 157 llvm::cl::desc( 158 "Use the preprocessor optimization that skips excluded conditionals by " 159 "bumping the buffer pointer in the lexer instead of lexing the tokens " 160 "until reaching the end directive."), 161 llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory)); 162 163 llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional, 164 llvm::cl::desc("Use verbose output."), 165 llvm::cl::init(false), 166 llvm::cl::cat(DependencyScannerCategory)); 167 168 } // end anonymous namespace 169 170 /// \returns object-file path derived from source-file path. 171 static std::string getObjFilePath(StringRef SrcFile) { 172 SmallString<128> ObjFileName(SrcFile); 173 llvm::sys::path::replace_extension(ObjFileName, "o"); 174 return std::string(ObjFileName.str()); 175 } 176 177 class SingleCommandCompilationDatabase : public tooling::CompilationDatabase { 178 public: 179 SingleCommandCompilationDatabase(tooling::CompileCommand Cmd) 180 : Command(std::move(Cmd)) {} 181 182 virtual std::vector<tooling::CompileCommand> 183 getCompileCommands(StringRef FilePath) const { 184 return {Command}; 185 } 186 187 virtual std::vector<tooling::CompileCommand> getAllCompileCommands() const { 188 return {Command}; 189 } 190 191 private: 192 tooling::CompileCommand Command; 193 }; 194 195 /// Takes the result of a dependency scan and prints error / dependency files 196 /// based on the result. 197 /// 198 /// \returns True on error. 199 static bool 200 handleMakeDependencyToolResult(const std::string &Input, 201 llvm::Expected<std::string> &MaybeFile, 202 SharedStream &OS, SharedStream &Errs) { 203 if (!MaybeFile) { 204 llvm::handleAllErrors( 205 MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) { 206 Errs.applyLocked([&](raw_ostream &OS) { 207 OS << "Error while scanning dependencies for " << Input << ":\n"; 208 OS << Err.getMessage(); 209 }); 210 }); 211 return true; 212 } 213 OS.applyLocked([&](raw_ostream &OS) { OS << *MaybeFile; }); 214 return false; 215 } 216 217 static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { 218 std::vector<llvm::StringRef> Strings; 219 for (auto &&I : Set) 220 Strings.push_back(I.getKey()); 221 llvm::sort(Strings); 222 return llvm::json::Array(Strings); 223 } 224 225 static llvm::json::Array toJSONSorted(std::vector<ClangModuleDep> V) { 226 llvm::sort(V, [](const ClangModuleDep &A, const ClangModuleDep &B) { 227 return std::tie(A.ModuleName, A.ContextHash) < 228 std::tie(B.ModuleName, B.ContextHash); 229 }); 230 231 llvm::json::Array Ret; 232 for (const ClangModuleDep &CMD : V) 233 Ret.push_back(llvm::json::Object( 234 {{"module-name", CMD.ModuleName}, {"context-hash", CMD.ContextHash}})); 235 return Ret; 236 } 237 238 // Thread safe. 239 class FullDeps { 240 public: 241 void mergeDeps(StringRef Input, FullDependenciesResult FDR, 242 size_t InputIndex) { 243 const FullDependencies &FD = FDR.FullDeps; 244 245 InputDeps ID; 246 ID.FileName = std::string(Input); 247 ID.ContextHash = std::move(FD.ContextHash); 248 ID.FileDeps = std::move(FD.FileDeps); 249 ID.ModuleDeps = std::move(FD.ClangModuleDeps); 250 251 std::unique_lock<std::mutex> ul(Lock); 252 for (const ModuleDeps &MD : FDR.DiscoveredModules) { 253 auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0}); 254 if (I != Modules.end()) { 255 I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); 256 continue; 257 } 258 Modules.insert( 259 I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)}); 260 } 261 262 if (FullCommandLine) 263 ID.AdditonalCommandLine = FD.getAdditionalCommandLine( 264 [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, 265 [&](ClangModuleDep CMD) -> const ModuleDeps & { 266 return lookupModuleDeps(CMD); 267 }); 268 269 Inputs.push_back(std::move(ID)); 270 } 271 272 void printFullOutput(raw_ostream &OS) { 273 // Sort the modules by name to get a deterministic order. 274 std::vector<ContextModulePair> ModuleNames; 275 for (auto &&M : Modules) 276 ModuleNames.push_back(M.first); 277 llvm::sort(ModuleNames, 278 [](const ContextModulePair &A, const ContextModulePair &B) { 279 return std::tie(A.ModuleName, A.InputIndex) < 280 std::tie(B.ModuleName, B.InputIndex); 281 }); 282 283 llvm::sort(Inputs, [](const InputDeps &A, const InputDeps &B) { 284 return A.FileName < B.FileName; 285 }); 286 287 using namespace llvm::json; 288 289 Array OutModules; 290 for (auto &&ModName : ModuleNames) { 291 auto &MD = Modules[ModName]; 292 Object O{ 293 {"name", MD.ModuleName}, 294 {"context-hash", MD.ContextHash}, 295 {"file-deps", toJSONSorted(MD.FileDeps)}, 296 {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, 297 {"clang-modulemap-file", MD.ClangModuleMapFile}, 298 {"command-line", 299 FullCommandLine 300 ? MD.getFullCommandLine( 301 [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, 302 [&](ClangModuleDep CMD) -> const ModuleDeps & { 303 return lookupModuleDeps(CMD); 304 }) 305 : MD.NonPathCommandLine}, 306 }; 307 OutModules.push_back(std::move(O)); 308 } 309 310 Array TUs; 311 for (auto &&I : Inputs) { 312 Object O{ 313 {"input-file", I.FileName}, 314 {"clang-context-hash", I.ContextHash}, 315 {"file-deps", I.FileDeps}, 316 {"clang-module-deps", toJSONSorted(I.ModuleDeps)}, 317 {"command-line", I.AdditonalCommandLine}, 318 }; 319 TUs.push_back(std::move(O)); 320 } 321 322 Object Output{ 323 {"modules", std::move(OutModules)}, 324 {"translation-units", std::move(TUs)}, 325 }; 326 327 OS << llvm::formatv("{0:2}\n", Value(std::move(Output))); 328 } 329 330 private: 331 StringRef lookupPCMPath(ClangModuleDep CMD) { 332 return Modules[ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}] 333 .ImplicitModulePCMPath; 334 } 335 336 const ModuleDeps &lookupModuleDeps(ClangModuleDep CMD) { 337 auto I = 338 Modules.find(ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}); 339 assert(I != Modules.end()); 340 return I->second; 341 }; 342 343 struct ContextModulePair { 344 std::string ContextHash; 345 std::string ModuleName; 346 mutable size_t InputIndex; 347 348 bool operator==(const ContextModulePair &Other) const { 349 return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName; 350 } 351 }; 352 353 struct ContextModulePairHasher { 354 std::size_t operator()(const ContextModulePair &CMP) const { 355 using llvm::hash_combine; 356 357 return hash_combine(CMP.ContextHash, CMP.ModuleName); 358 } 359 }; 360 361 struct InputDeps { 362 std::string FileName; 363 std::string ContextHash; 364 std::vector<std::string> FileDeps; 365 std::vector<ClangModuleDep> ModuleDeps; 366 std::vector<std::string> AdditonalCommandLine; 367 }; 368 369 std::mutex Lock; 370 std::unordered_map<ContextModulePair, ModuleDeps, ContextModulePairHasher> 371 Modules; 372 std::vector<InputDeps> Inputs; 373 }; 374 375 static bool handleFullDependencyToolResult( 376 const std::string &Input, 377 llvm::Expected<FullDependenciesResult> &MaybeFullDeps, FullDeps &FD, 378 size_t InputIndex, SharedStream &OS, SharedStream &Errs) { 379 if (!MaybeFullDeps) { 380 llvm::handleAllErrors( 381 MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) { 382 Errs.applyLocked([&](raw_ostream &OS) { 383 OS << "Error while scanning dependencies for " << Input << ":\n"; 384 OS << Err.getMessage(); 385 }); 386 }); 387 return true; 388 } 389 FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex); 390 return false; 391 } 392 393 int main(int argc, const char **argv) { 394 llvm::InitLLVM X(argc, argv); 395 llvm::cl::HideUnrelatedOptions(DependencyScannerCategory); 396 if (!llvm::cl::ParseCommandLineOptions(argc, argv)) 397 return 1; 398 399 std::string ErrorMessage; 400 std::unique_ptr<tooling::JSONCompilationDatabase> Compilations = 401 tooling::JSONCompilationDatabase::loadFromFile( 402 CompilationDB, ErrorMessage, 403 tooling::JSONCommandLineSyntax::AutoDetect); 404 if (!Compilations) { 405 llvm::errs() << "error: " << ErrorMessage << "\n"; 406 return 1; 407 } 408 409 llvm::cl::PrintOptionValues(); 410 411 // The command options are rewritten to run Clang in preprocessor only mode. 412 auto AdjustingCompilations = 413 std::make_unique<tooling::ArgumentsAdjustingCompilations>( 414 std::move(Compilations)); 415 ResourceDirectoryCache ResourceDirCache; 416 AdjustingCompilations->appendArgumentsAdjuster( 417 [&ResourceDirCache](const tooling::CommandLineArguments &Args, 418 StringRef FileName) { 419 std::string LastO = ""; 420 bool HasMT = false; 421 bool HasMQ = false; 422 bool HasMD = false; 423 bool HasResourceDir = false; 424 // We need to find the last -o value. 425 if (!Args.empty()) { 426 std::size_t Idx = Args.size() - 1; 427 for (auto It = Args.rbegin(); It != Args.rend(); ++It) { 428 if (It != Args.rbegin()) { 429 if (Args[Idx] == "-o") 430 LastO = Args[Idx + 1]; 431 if (Args[Idx] == "-MT") 432 HasMT = true; 433 if (Args[Idx] == "-MQ") 434 HasMQ = true; 435 if (Args[Idx] == "-MD") 436 HasMD = true; 437 if (Args[Idx] == "-resource-dir") 438 HasResourceDir = true; 439 } 440 --Idx; 441 } 442 } 443 // If there's no -MT/-MQ Driver would add -MT with the value of the last 444 // -o option. 445 tooling::CommandLineArguments AdjustedArgs = Args; 446 AdjustedArgs.push_back("-o"); 447 AdjustedArgs.push_back("/dev/null"); 448 if (!HasMT && !HasMQ) { 449 AdjustedArgs.push_back("-M"); 450 AdjustedArgs.push_back("-MT"); 451 // We're interested in source dependencies of an object file. 452 if (!HasMD) { 453 // FIXME: We are missing the directory unless the -o value is an 454 // absolute path. 455 AdjustedArgs.push_back(!LastO.empty() ? LastO 456 : getObjFilePath(FileName)); 457 } else { 458 AdjustedArgs.push_back(std::string(FileName)); 459 } 460 } 461 AdjustedArgs.push_back("-Xclang"); 462 AdjustedArgs.push_back("-Eonly"); 463 AdjustedArgs.push_back("-Xclang"); 464 AdjustedArgs.push_back("-sys-header-deps"); 465 AdjustedArgs.push_back("-Wno-error"); 466 467 if (!HasResourceDir) { 468 StringRef ResourceDir = 469 ResourceDirCache.findResourceDir(Args); 470 if (!ResourceDir.empty()) { 471 AdjustedArgs.push_back("-resource-dir"); 472 AdjustedArgs.push_back(std::string(ResourceDir)); 473 } 474 } 475 return AdjustedArgs; 476 }); 477 AdjustingCompilations->appendArgumentsAdjuster( 478 tooling::getClangStripSerializeDiagnosticAdjuster()); 479 480 SharedStream Errs(llvm::errs()); 481 // Print out the dependency results to STDOUT by default. 482 SharedStream DependencyOS(llvm::outs()); 483 484 DependencyScanningService Service(ScanMode, Format, ReuseFileManager, 485 SkipExcludedPPRanges); 486 llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); 487 std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools; 488 for (unsigned I = 0; I < Pool.getThreadCount(); ++I) 489 WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service)); 490 491 std::vector<SingleCommandCompilationDatabase> Inputs; 492 for (tooling::CompileCommand Cmd : 493 AdjustingCompilations->getAllCompileCommands()) 494 Inputs.emplace_back(Cmd); 495 496 std::atomic<bool> HadErrors(false); 497 FullDeps FD; 498 std::mutex Lock; 499 size_t Index = 0; 500 501 if (Verbose) { 502 llvm::outs() << "Running clang-scan-deps on " << Inputs.size() 503 << " files using " << Pool.getThreadCount() << " workers\n"; 504 } 505 for (unsigned I = 0; I < Pool.getThreadCount(); ++I) { 506 Pool.async([I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, 507 &DependencyOS, &Errs]() { 508 llvm::StringSet<> AlreadySeenModules; 509 while (true) { 510 const SingleCommandCompilationDatabase *Input; 511 std::string Filename; 512 std::string CWD; 513 size_t LocalIndex; 514 // Take the next input. 515 { 516 std::unique_lock<std::mutex> LockGuard(Lock); 517 if (Index >= Inputs.size()) 518 return; 519 LocalIndex = Index; 520 Input = &Inputs[Index++]; 521 tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0]; 522 Filename = std::move(Cmd.Filename); 523 CWD = std::move(Cmd.Directory); 524 } 525 // Run the tool on it. 526 if (Format == ScanningOutputFormat::Make) { 527 auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); 528 if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, 529 Errs)) 530 HadErrors = true; 531 } else { 532 auto MaybeFullDeps = WorkerTools[I]->getFullDependencies( 533 *Input, CWD, AlreadySeenModules); 534 if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD, 535 LocalIndex, DependencyOS, Errs)) 536 HadErrors = true; 537 } 538 } 539 }); 540 } 541 Pool.wait(); 542 543 if (Format == ScanningOutputFormat::Full) 544 FD.printFullOutput(llvm::outs()); 545 546 return HadErrors; 547 } 548