1 //===--- ParsedAST.cpp -------------------------------------------*- C++-*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "ParsedAST.h" 10 #include "../clang-tidy/ClangTidyCheck.h" 11 #include "../clang-tidy/ClangTidyDiagnosticConsumer.h" 12 #include "../clang-tidy/ClangTidyModule.h" 13 #include "../clang-tidy/ClangTidyModuleRegistry.h" 14 #include "../clang-tidy/ClangTidyOptions.h" 15 #include "AST.h" 16 #include "CollectMacros.h" 17 #include "Compiler.h" 18 #include "Config.h" 19 #include "Diagnostics.h" 20 #include "Feature.h" 21 #include "FeatureModule.h" 22 #include "Headers.h" 23 #include "IncludeCleaner.h" 24 #include "IncludeFixer.h" 25 #include "Preamble.h" 26 #include "SourceCode.h" 27 #include "TidyProvider.h" 28 #include "clang-include-cleaner/Record.h" 29 #include "index/Symbol.h" 30 #include "support/Logger.h" 31 #include "support/Path.h" 32 #include "support/Trace.h" 33 #include "clang/AST/ASTContext.h" 34 #include "clang/AST/Decl.h" 35 #include "clang/AST/DeclGroup.h" 36 #include "clang/AST/ExternalASTSource.h" 37 #include "clang/ASTMatchers/ASTMatchFinder.h" 38 #include "clang/Basic/Diagnostic.h" 39 #include "clang/Basic/DiagnosticIDs.h" 40 #include "clang/Basic/DiagnosticSema.h" 41 #include "clang/Basic/FileEntry.h" 42 #include "clang/Basic/LLVM.h" 43 #include "clang/Basic/LangOptions.h" 44 #include "clang/Basic/SourceLocation.h" 45 #include "clang/Basic/SourceManager.h" 46 #include "clang/Basic/TokenKinds.h" 47 #include "clang/Frontend/CompilerInstance.h" 48 #include "clang/Frontend/CompilerInvocation.h" 49 #include "clang/Frontend/FrontendActions.h" 50 #include "clang/Frontend/FrontendOptions.h" 51 #include "clang/Frontend/PrecompiledPreamble.h" 52 #include "clang/Lex/Lexer.h" 53 #include "clang/Lex/PPCallbacks.h" 54 #include "clang/Lex/Preprocessor.h" 55 #include "clang/Sema/HeuristicResolver.h" 56 #include "clang/Serialization/ASTWriter.h" 57 #include "clang/Tooling/CompilationDatabase.h" 58 #include "clang/Tooling/Core/Diagnostic.h" 59 #include "clang/Tooling/Syntax/Tokens.h" 60 #include "llvm/ADT/ArrayRef.h" 61 #include "llvm/ADT/DenseMap.h" 62 #include "llvm/ADT/DenseSet.h" 63 #include "llvm/ADT/STLExtras.h" 64 #include "llvm/ADT/STLFunctionalExtras.h" 65 #include "llvm/ADT/SmallVector.h" 66 #include "llvm/ADT/StringRef.h" 67 #include "llvm/Support/Error.h" 68 #include "llvm/Support/MemoryBuffer.h" 69 #include <cassert> 70 #include <cstddef> 71 #include <iterator> 72 #include <memory> 73 #include <optional> 74 #include <string> 75 #include <tuple> 76 #include <utility> 77 #include <vector> 78 79 // Force the linker to link in Clang-tidy modules. 80 // clangd doesn't support the static analyzer. 81 #if CLANGD_TIDY_CHECKS 82 #define CLANG_TIDY_DISABLE_STATIC_ANALYZER_CHECKS 83 #include "../clang-tidy/ClangTidyForceLinker.h" 84 #endif 85 86 namespace clang { 87 namespace clangd { 88 namespace { 89 90 template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) { 91 return Vec.capacity() * sizeof(T); 92 } 93 94 class DeclTrackingASTConsumer : public ASTConsumer { 95 public: 96 DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls) 97 : TopLevelDecls(TopLevelDecls) {} 98 99 bool HandleTopLevelDecl(DeclGroupRef DG) override { 100 for (Decl *D : DG) { 101 auto &SM = D->getASTContext().getSourceManager(); 102 if (!isInsideMainFile(D->getLocation(), SM)) 103 continue; 104 if (const NamedDecl *ND = dyn_cast<NamedDecl>(D)) 105 if (isImplicitTemplateInstantiation(ND)) 106 continue; 107 108 // ObjCMethodDecl are not actually top-level decls. 109 if (isa<ObjCMethodDecl>(D)) 110 continue; 111 112 TopLevelDecls.push_back(D); 113 } 114 return true; 115 } 116 117 private: 118 std::vector<Decl *> &TopLevelDecls; 119 }; 120 121 class ClangdFrontendAction : public SyntaxOnlyAction { 122 public: 123 std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); } 124 125 protected: 126 std::unique_ptr<ASTConsumer> 127 CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override { 128 return std::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls); 129 } 130 131 private: 132 std::vector<Decl *> TopLevelDecls; 133 }; 134 135 // When using a preamble, only preprocessor events outside its bounds are seen. 136 // This is almost what we want: replaying transitive preprocessing wastes time. 137 // However this confuses clang-tidy checks: they don't see any #includes! 138 // So we replay the *non-transitive* #includes that appear in the main-file. 139 // It would be nice to replay other events (macro definitions, ifdefs etc) but 140 // this addresses the most common cases fairly cheaply. 141 class ReplayPreamble : private PPCallbacks { 142 public: 143 // Attach preprocessor hooks such that preamble events will be injected at 144 // the appropriate time. 145 // Events will be delivered to the *currently registered* PP callbacks. 146 static void attach(std::vector<Inclusion> Includes, CompilerInstance &Clang, 147 const PreambleBounds &PB) { 148 auto &PP = Clang.getPreprocessor(); 149 auto *ExistingCallbacks = PP.getPPCallbacks(); 150 // No need to replay events if nobody is listening. 151 if (!ExistingCallbacks) 152 return; 153 PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(new ReplayPreamble( 154 std::move(Includes), ExistingCallbacks, Clang.getSourceManager(), PP, 155 Clang.getLangOpts(), PB))); 156 // We're relying on the fact that addPPCallbacks keeps the old PPCallbacks 157 // around, creating a chaining wrapper. Guard against other implementations. 158 assert(PP.getPPCallbacks() != ExistingCallbacks && 159 "Expected chaining implementation"); 160 } 161 162 private: 163 ReplayPreamble(std::vector<Inclusion> Includes, PPCallbacks *Delegate, 164 const SourceManager &SM, Preprocessor &PP, 165 const LangOptions &LangOpts, const PreambleBounds &PB) 166 : Includes(std::move(Includes)), Delegate(Delegate), SM(SM), PP(PP) { 167 // Only tokenize the preamble section of the main file, as we are not 168 // interested in the rest of the tokens. 169 MainFileTokens = syntax::tokenize( 170 syntax::FileRange(SM.getMainFileID(), 0, PB.Size), SM, LangOpts); 171 } 172 173 // In a normal compile, the preamble traverses the following structure: 174 // 175 // mainfile.cpp 176 // <built-in> 177 // ... macro definitions like __cplusplus ... 178 // <command-line> 179 // ... macro definitions for args like -Dfoo=bar ... 180 // "header1.h" 181 // ... header file contents ... 182 // "header2.h" 183 // ... header file contents ... 184 // ... main file contents ... 185 // 186 // When using a preamble, the "header1" and "header2" subtrees get skipped. 187 // We insert them right after the built-in header, which still appears. 188 void FileChanged(SourceLocation Loc, FileChangeReason Reason, 189 SrcMgr::CharacteristicKind Kind, FileID PrevFID) override { 190 // It'd be nice if there was a better way to identify built-in headers... 191 if (Reason == FileChangeReason::ExitFile && 192 SM.getBufferOrFake(PrevFID).getBufferIdentifier() == "<built-in>") 193 replay(); 194 } 195 196 void replay() { 197 for (const auto &Inc : Includes) { 198 OptionalFileEntryRef File; 199 if (Inc.Resolved != "") 200 File = expectedToOptional(SM.getFileManager().getFileRef(Inc.Resolved)); 201 202 // Re-lex the #include directive to find its interesting parts. 203 auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset); 204 auto HashTok = llvm::partition_point(MainFileTokens, 205 [&HashLoc](const syntax::Token &T) { 206 return T.location() < HashLoc; 207 }); 208 assert(HashTok != MainFileTokens.end() && HashTok->kind() == tok::hash); 209 210 auto IncludeTok = std::next(HashTok); 211 assert(IncludeTok != MainFileTokens.end()); 212 213 auto FileTok = std::next(IncludeTok); 214 assert(FileTok != MainFileTokens.end()); 215 216 // Create a fake import/include token, none of the callers seem to care 217 // about clang::Token::Flags. 218 Token SynthesizedIncludeTok; 219 SynthesizedIncludeTok.startToken(); 220 SynthesizedIncludeTok.setLocation(IncludeTok->location()); 221 SynthesizedIncludeTok.setLength(IncludeTok->length()); 222 SynthesizedIncludeTok.setKind(tok::raw_identifier); 223 SynthesizedIncludeTok.setRawIdentifierData(IncludeTok->text(SM).data()); 224 PP.LookUpIdentifierInfo(SynthesizedIncludeTok); 225 226 // Same here, create a fake one for Filename, including angles or quotes. 227 Token SynthesizedFilenameTok; 228 SynthesizedFilenameTok.startToken(); 229 SynthesizedFilenameTok.setLocation(FileTok->location()); 230 // Note that we can't make use of FileTok->length/text in here as in the 231 // case of angled includes this will contain tok::less instead of 232 // filename. Whereas Inc.Written contains the full header name including 233 // quotes/angles. 234 SynthesizedFilenameTok.setLength(Inc.Written.length()); 235 SynthesizedFilenameTok.setKind(tok::header_name); 236 SynthesizedFilenameTok.setLiteralData(Inc.Written.data()); 237 238 llvm::StringRef WrittenFilename = 239 llvm::StringRef(Inc.Written).drop_front().drop_back(); 240 Delegate->InclusionDirective( 241 HashTok->location(), SynthesizedIncludeTok, WrittenFilename, 242 Inc.Written.front() == '<', 243 syntax::FileRange(SM, SynthesizedFilenameTok.getLocation(), 244 SynthesizedFilenameTok.getEndLoc()) 245 .toCharRange(SM), 246 File, "SearchPath", "RelPath", 247 /*SuggestedModule=*/nullptr, /*ModuleImported=*/false, Inc.FileKind); 248 if (File) 249 Delegate->FileSkipped(*File, SynthesizedFilenameTok, Inc.FileKind); 250 } 251 } 252 253 const std::vector<Inclusion> Includes; 254 PPCallbacks *Delegate; 255 const SourceManager &SM; 256 Preprocessor &PP; 257 std::vector<syntax::Token> MainFileTokens; 258 }; 259 260 // Filter for clang diagnostics groups enabled by CTOptions.Checks. 261 // 262 // These are check names like clang-diagnostics-unused. 263 // Note that unlike -Wunused, clang-diagnostics-unused does not imply 264 // subcategories like clang-diagnostics-unused-function. 265 // 266 // This is used to determine which diagnostics can be enabled by ExtraArgs in 267 // the clang-tidy configuration. 268 class TidyDiagnosticGroups { 269 // Whether all diagnostic groups are enabled by default. 270 // True if we've seen clang-diagnostic-*. 271 bool Default = false; 272 // Set of diag::Group whose enablement != Default. 273 // If Default is false, this is foo where we've seen clang-diagnostic-foo. 274 llvm::DenseSet<unsigned> Exceptions; 275 276 public: 277 TidyDiagnosticGroups(llvm::StringRef Checks) { 278 constexpr llvm::StringLiteral CDPrefix = "clang-diagnostic-"; 279 280 llvm::StringRef Check; 281 while (!Checks.empty()) { 282 std::tie(Check, Checks) = Checks.split(','); 283 Check = Check.trim(); 284 285 if (Check.empty()) 286 continue; 287 288 bool Enable = !Check.consume_front("-"); 289 bool Glob = Check.consume_back("*"); 290 if (Glob) { 291 // Is this clang-diagnostic-*, or *, or so? 292 // (We ignore all other types of globs). 293 if (CDPrefix.starts_with(Check)) { 294 Default = Enable; 295 Exceptions.clear(); 296 } 297 continue; 298 } 299 300 // In "*,clang-diagnostic-foo", the latter is a no-op. 301 if (Default == Enable) 302 continue; 303 // The only non-glob entries we care about are clang-diagnostic-foo. 304 if (!Check.consume_front(CDPrefix)) 305 continue; 306 307 if (auto Group = DiagnosticIDs::getGroupForWarningOption(Check)) 308 Exceptions.insert(static_cast<unsigned>(*Group)); 309 } 310 } 311 312 bool operator()(diag::Group GroupID) const { 313 return Exceptions.contains(static_cast<unsigned>(GroupID)) ? !Default 314 : Default; 315 } 316 }; 317 318 // Find -W<group> and -Wno-<group> options in ExtraArgs and apply them to Diags. 319 // 320 // This is used to handle ExtraArgs in clang-tidy configuration. 321 // We don't use clang's standard handling of this as we want slightly different 322 // behavior (e.g. we want to exclude these from -Wno-error). 323 void applyWarningOptions(llvm::ArrayRef<std::string> ExtraArgs, 324 llvm::function_ref<bool(diag::Group)> EnabledGroups, 325 DiagnosticsEngine &Diags) { 326 for (llvm::StringRef Group : ExtraArgs) { 327 // Only handle args that are of the form -W[no-]<group>. 328 // Other flags are possible but rare and deliberately out of scope. 329 llvm::SmallVector<diag::kind> Members; 330 if (!Group.consume_front("-W") || Group.empty()) 331 continue; 332 bool Enable = !Group.consume_front("no-"); 333 if (Diags.getDiagnosticIDs()->getDiagnosticsInGroup( 334 diag::Flavor::WarningOrError, Group, Members)) 335 continue; 336 337 // Upgrade (or downgrade) the severity of each diagnostic in the group. 338 // If -Werror is on, newly added warnings will be treated as errors. 339 // We don't want this, so keep track of them to fix afterwards. 340 bool NeedsWerrorExclusion = false; 341 for (diag::kind ID : Members) { 342 if (Enable) { 343 if (Diags.getDiagnosticLevel(ID, SourceLocation()) < 344 DiagnosticsEngine::Warning) { 345 auto Group = Diags.getDiagnosticIDs()->getGroupForDiag(ID); 346 if (!Group || !EnabledGroups(*Group)) 347 continue; 348 Diags.setSeverity(ID, diag::Severity::Warning, SourceLocation()); 349 if (Diags.getWarningsAsErrors()) 350 NeedsWerrorExclusion = true; 351 } 352 } else { 353 Diags.setSeverity(ID, diag::Severity::Ignored, SourceLocation()); 354 } 355 } 356 if (NeedsWerrorExclusion) { 357 // FIXME: there's no API to suppress -Werror for single diagnostics. 358 // In some cases with sub-groups, we may end up erroneously 359 // downgrading diagnostics that were -Werror in the compile command. 360 Diags.setDiagnosticGroupWarningAsError(Group, false); 361 } 362 } 363 } 364 365 std::vector<Diag> getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code, 366 const ThreadsafeFS &TFS) { 367 auto &Cfg = Config::current(); 368 if (Cfg.Diagnostics.SuppressAll) 369 return {}; 370 bool SuppressMissing = 371 Cfg.Diagnostics.Suppress.contains("missing-includes") || 372 Cfg.Diagnostics.MissingIncludes == Config::IncludesPolicy::None; 373 bool SuppressUnused = 374 Cfg.Diagnostics.Suppress.contains("unused-includes") || 375 Cfg.Diagnostics.UnusedIncludes == Config::IncludesPolicy::None; 376 if (SuppressMissing && SuppressUnused) 377 return {}; 378 auto Findings = computeIncludeCleanerFindings( 379 AST, Cfg.Diagnostics.Includes.AnalyzeAngledIncludes); 380 if (SuppressMissing) 381 Findings.MissingIncludes.clear(); 382 if (SuppressUnused) 383 Findings.UnusedIncludes.clear(); 384 return issueIncludeCleanerDiagnostics(AST, Code, Findings, TFS, 385 Cfg.Diagnostics.Includes.IgnoreHeader); 386 } 387 388 tidy::ClangTidyCheckFactories 389 filterFastTidyChecks(const tidy::ClangTidyCheckFactories &All, 390 Config::FastCheckPolicy Policy) { 391 if (Policy == Config::FastCheckPolicy::None) 392 return All; 393 bool AllowUnknown = Policy == Config::FastCheckPolicy::Loose; 394 tidy::ClangTidyCheckFactories Fast; 395 for (const auto &Factory : All) { 396 if (isFastTidyCheck(Factory.getKey()).value_or(AllowUnknown)) 397 Fast.registerCheckFactory(Factory.first(), Factory.second); 398 } 399 return Fast; 400 } 401 402 } // namespace 403 404 std::optional<ParsedAST> 405 ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, 406 std::unique_ptr<clang::CompilerInvocation> CI, 407 llvm::ArrayRef<Diag> CompilerInvocationDiags, 408 std::shared_ptr<const PreambleData> Preamble) { 409 trace::Span Tracer("BuildAST"); 410 SPAN_ATTACH(Tracer, "File", Filename); 411 const Config &Cfg = Config::current(); 412 413 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory); 414 if (Preamble && Preamble->StatCache) 415 VFS = Preamble->StatCache->getConsumingFS(std::move(VFS)); 416 417 assert(CI); 418 419 if (CI->getFrontendOpts().Inputs.size() > 0) { 420 auto Lang = CI->getFrontendOpts().Inputs[0].getKind().getLanguage(); 421 if (Lang == Language::Asm || Lang == Language::LLVM_IR) { 422 elog("Clangd does not support assembly or IR source files"); 423 return std::nullopt; 424 } 425 } 426 427 // Command-line parsing sets DisableFree to true by default, but we don't want 428 // to leak memory in clangd. 429 CI->getFrontendOpts().DisableFree = false; 430 const PrecompiledPreamble *PreamblePCH = 431 Preamble ? &Preamble->Preamble : nullptr; 432 433 // This is on-by-default in windows to allow parsing SDK headers, but it 434 // breaks many features. Disable it for the main-file (not preamble). 435 CI->getLangOpts().DelayedTemplateParsing = false; 436 437 std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners; 438 if (Inputs.FeatureModules) { 439 for (auto &M : *Inputs.FeatureModules) { 440 if (auto Listener = M.astListeners()) 441 ASTListeners.emplace_back(std::move(Listener)); 442 } 443 } 444 StoreDiags ASTDiags; 445 ASTDiags.setDiagCallback( 446 [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) { 447 for (const auto &L : ASTListeners) 448 L->sawDiagnostic(D, Diag); 449 }); 450 451 // Adjust header search options to load the built module files recorded 452 // in RequiredModules. 453 if (Preamble && Preamble->RequiredModules) 454 Preamble->RequiredModules->adjustHeaderSearchOptions( 455 CI->getHeaderSearchOpts()); 456 457 std::optional<PreamblePatch> Patch; 458 // We might use an ignoring diagnostic consumer if they are going to be 459 // dropped later on to not pay for extra latency by processing them. 460 DiagnosticConsumer *DiagConsumer = &ASTDiags; 461 IgnoreDiagnostics DropDiags; 462 if (Preamble) { 463 Patch = PreamblePatch::createFullPatch(Filename, Inputs, *Preamble); 464 Patch->apply(*CI); 465 } 466 auto Clang = prepareCompilerInstance( 467 std::move(CI), PreamblePCH, 468 llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS, 469 *DiagConsumer); 470 471 if (!Clang) { 472 // The last diagnostic contains information about the reason of this 473 // failure. 474 std::vector<Diag> Diags(ASTDiags.take()); 475 elog("Failed to prepare a compiler instance: {0}", 476 !Diags.empty() ? static_cast<DiagBase &>(Diags.back()).Message 477 : "unknown error"); 478 return std::nullopt; 479 } 480 tidy::ClangTidyOptions ClangTidyOpts; 481 { 482 trace::Span Tracer("ClangTidyOpts"); 483 ClangTidyOpts = getTidyOptionsForFile(Inputs.ClangTidyProvider, Filename); 484 dlog("ClangTidy configuration for file {0}: {1}", Filename, 485 tidy::configurationAsText(ClangTidyOpts)); 486 487 // If clang-tidy is configured to emit clang warnings, we should too. 488 // 489 // Such clang-tidy configuration consists of two parts: 490 // - ExtraArgs: ["-Wfoo"] causes clang to produce the warnings 491 // - Checks: "clang-diagnostic-foo" prevents clang-tidy filtering them out 492 // 493 // In clang-tidy, diagnostics are emitted if they pass both checks. 494 // When groups contain subgroups, -Wparent includes the child, but 495 // clang-diagnostic-parent does not. 496 // 497 // We *don't* want to change the compile command directly. This can have 498 // too many unexpected effects: breaking the command, interactions with 499 // -- and -Werror, etc. Besides, we've already parsed the command. 500 // Instead we parse the -W<group> flags and handle them directly. 501 // 502 // Similarly, we don't want to use Checks to filter clang diagnostics after 503 // they are generated, as this spreads clang-tidy emulation everywhere. 504 // Instead, we just use these to filter which extra diagnostics we enable. 505 auto &Diags = Clang->getDiagnostics(); 506 TidyDiagnosticGroups TidyGroups(ClangTidyOpts.Checks ? *ClangTidyOpts.Checks 507 : llvm::StringRef()); 508 if (ClangTidyOpts.ExtraArgsBefore) 509 applyWarningOptions(*ClangTidyOpts.ExtraArgsBefore, TidyGroups, Diags); 510 if (ClangTidyOpts.ExtraArgs) 511 applyWarningOptions(*ClangTidyOpts.ExtraArgs, TidyGroups, Diags); 512 } 513 514 auto Action = std::make_unique<ClangdFrontendAction>(); 515 const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0]; 516 if (!Action->BeginSourceFile(*Clang, MainInput)) { 517 elog("BeginSourceFile() failed when building AST for {0}", 518 MainInput.getFile()); 519 return std::nullopt; 520 } 521 // If we saw an include guard in the preamble section of the main file, 522 // mark the main-file as include-guarded. 523 // This information is part of the HeaderFileInfo but is not loaded from the 524 // preamble as the file's size is part of its identity and may have changed. 525 // (The rest of HeaderFileInfo is not relevant for our purposes). 526 if (Preamble && Preamble->MainIsIncludeGuarded) { 527 const SourceManager &SM = Clang->getSourceManager(); 528 OptionalFileEntryRef MainFE = SM.getFileEntryRefForID(SM.getMainFileID()); 529 Clang->getPreprocessor().getHeaderSearchInfo().MarkFileIncludeOnce(*MainFE); 530 } 531 532 // Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists. 533 // Clang-tidy has some limitations to ensure reasonable performance: 534 // - checks don't see all preprocessor events in the preamble 535 // - matchers run only over the main-file top-level decls (and can't see 536 // ancestors outside this scope). 537 // In practice almost all checks work well without modifications. 538 std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks; 539 ast_matchers::MatchFinder CTFinder; 540 std::optional<tidy::ClangTidyContext> CTContext; 541 // Must outlive FixIncludes. 542 auto BuildDir = VFS->getCurrentWorkingDirectory(); 543 std::optional<IncludeFixer> FixIncludes; 544 llvm::DenseMap<diag::kind, DiagnosticsEngine::Level> OverriddenSeverity; 545 // No need to run clang-tidy or IncludeFixerif we are not going to surface 546 // diagnostics. 547 { 548 trace::Span Tracer("ClangTidyInit"); 549 static const auto *AllCTFactories = [] { 550 auto *CTFactories = new tidy::ClangTidyCheckFactories; 551 for (const auto &E : tidy::ClangTidyModuleRegistry::entries()) 552 E.instantiate()->addCheckFactories(*CTFactories); 553 return CTFactories; 554 }(); 555 tidy::ClangTidyCheckFactories FastFactories = filterFastTidyChecks( 556 *AllCTFactories, Cfg.Diagnostics.ClangTidy.FastCheckFilter); 557 CTContext.emplace(std::make_unique<tidy::DefaultOptionsProvider>( 558 tidy::ClangTidyGlobalOptions(), ClangTidyOpts)); 559 CTContext->setDiagnosticsEngine(&Clang->getDiagnostics()); 560 CTContext->setASTContext(&Clang->getASTContext()); 561 CTContext->setCurrentFile(Filename); 562 CTContext->setSelfContainedDiags(true); 563 CTChecks = FastFactories.createChecksForLanguage(&*CTContext); 564 Preprocessor *PP = &Clang->getPreprocessor(); 565 for (const auto &Check : CTChecks) { 566 Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP); 567 Check->registerMatchers(&CTFinder); 568 } 569 570 // Clang only corrects typos for use of undeclared functions in C if that 571 // use is an error. Include fixer relies on typo correction, so pretend 572 // this is an error. (The actual typo correction is nice too). 573 // We restore the original severity in the level adjuster. 574 // FIXME: It would be better to have a real API for this, but what? 575 for (auto ID : {diag::ext_implicit_function_decl_c99, 576 diag::ext_implicit_lib_function_decl, 577 diag::ext_implicit_lib_function_decl_c99, 578 diag::warn_implicit_function_decl}) { 579 OverriddenSeverity.try_emplace( 580 ID, Clang->getDiagnostics().getDiagnosticLevel(ID, SourceLocation())); 581 Clang->getDiagnostics().setSeverity(ID, diag::Severity::Error, 582 SourceLocation()); 583 } 584 585 ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel, 586 const clang::Diagnostic &Info) { 587 if (Cfg.Diagnostics.SuppressAll || 588 isDiagnosticSuppressed(Info, Cfg.Diagnostics.Suppress, 589 Clang->getLangOpts())) 590 return DiagnosticsEngine::Ignored; 591 592 auto It = OverriddenSeverity.find(Info.getID()); 593 if (It != OverriddenSeverity.end()) 594 DiagLevel = It->second; 595 596 if (!CTChecks.empty()) { 597 std::string CheckName = CTContext->getCheckName(Info.getID()); 598 bool IsClangTidyDiag = !CheckName.empty(); 599 if (IsClangTidyDiag) { 600 if (Cfg.Diagnostics.Suppress.contains(CheckName)) 601 return DiagnosticsEngine::Ignored; 602 // Check for suppression comment. Skip the check for diagnostics not 603 // in the main file, because we don't want that function to query the 604 // source buffer for preamble files. For the same reason, we ask 605 // shouldSuppressDiagnostic to avoid I/O. 606 // We let suppression comments take precedence over warning-as-error 607 // to match clang-tidy's behaviour. 608 bool IsInsideMainFile = 609 Info.hasSourceManager() && 610 isInsideMainFile(Info.getLocation(), Info.getSourceManager()); 611 SmallVector<tooling::Diagnostic, 1> TidySuppressedErrors; 612 if (IsInsideMainFile && CTContext->shouldSuppressDiagnostic( 613 DiagLevel, Info, TidySuppressedErrors, 614 /*AllowIO=*/false, 615 /*EnableNolintBlocks=*/true)) { 616 // FIXME: should we expose the suppression error (invalid use of 617 // NOLINT comments)? 618 return DiagnosticsEngine::Ignored; 619 } 620 if (!CTContext->getOptions().SystemHeaders.value_or(false) && 621 Info.hasSourceManager() && 622 Info.getSourceManager().isInSystemMacro(Info.getLocation())) 623 return DiagnosticsEngine::Ignored; 624 625 // Check for warning-as-error. 626 if (DiagLevel == DiagnosticsEngine::Warning && 627 CTContext->treatAsError(CheckName)) { 628 return DiagnosticsEngine::Error; 629 } 630 } 631 } 632 return DiagLevel; 633 }); 634 635 // Add IncludeFixer which can recover diagnostics caused by missing includes 636 // (e.g. incomplete type) and attach include insertion fixes to diagnostics. 637 if (Inputs.Index && !BuildDir.getError()) { 638 auto Style = 639 getFormatStyleForFile(Filename, Inputs.Contents, *Inputs.TFS, false); 640 auto Inserter = std::make_shared<IncludeInserter>( 641 Filename, Inputs.Contents, Style, BuildDir.get(), 642 &Clang->getPreprocessor().getHeaderSearchInfo(), 643 Cfg.Style.QuotedHeaders, Cfg.Style.AngledHeaders); 644 ArrayRef<Inclusion> MainFileIncludes; 645 if (Preamble) { 646 MainFileIncludes = Preamble->Includes.MainFileIncludes; 647 for (const auto &Inc : Preamble->Includes.MainFileIncludes) 648 Inserter->addExisting(Inc); 649 } 650 // FIXME: Consider piping through ASTSignals to fetch this to handle the 651 // case where a header file contains ObjC decls but no #imports. 652 Symbol::IncludeDirective Directive = 653 Inputs.Opts.ImportInsertions 654 ? preferredIncludeDirective(Filename, Clang->getLangOpts(), 655 MainFileIncludes, {}) 656 : Symbol::Include; 657 FixIncludes.emplace(Filename, Inserter, *Inputs.Index, 658 /*IndexRequestLimit=*/5, Directive); 659 ASTDiags.contributeFixes([&FixIncludes](DiagnosticsEngine::Level DiagLevl, 660 const clang::Diagnostic &Info) { 661 return FixIncludes->fix(DiagLevl, Info); 662 }); 663 Clang->setExternalSemaSource(FixIncludes->unresolvedNameRecorder()); 664 } 665 } 666 667 IncludeStructure Includes; 668 include_cleaner::PragmaIncludes PI; 669 // If we are using a preamble, copy existing includes. 670 if (Preamble) { 671 Includes = Preamble->Includes; 672 Includes.MainFileIncludes = Patch->preambleIncludes(); 673 // Replay the preamble includes so that clang-tidy checks can see them. 674 ReplayPreamble::attach(Patch->preambleIncludes(), *Clang, 675 Patch->modifiedBounds()); 676 PI = *Preamble->Pragmas; 677 } 678 // Important: collectIncludeStructure is registered *after* ReplayPreamble! 679 // Otherwise we would collect the replayed includes again... 680 // (We can't *just* use the replayed includes, they don't have Resolved path). 681 Includes.collect(*Clang); 682 // Same for pragma-includes, we're already inheriting preamble includes, so we 683 // should only receive callbacks for non-preamble mainfile includes. 684 PI.record(*Clang); 685 // Copy over the macros in the preamble region of the main file, and combine 686 // with non-preamble macros below. 687 MainFileMacros Macros; 688 std::vector<PragmaMark> Marks; 689 if (Preamble) { 690 Macros = Patch->mainFileMacros(); 691 Marks = Patch->marks(); 692 } 693 auto &PP = Clang->getPreprocessor(); 694 auto MacroCollector = std::make_unique<CollectMainFileMacros>(PP, Macros); 695 auto *MacroCollectorPtr = MacroCollector.get(); // so we can call doneParse() 696 PP.addPPCallbacks(std::move(MacroCollector)); 697 698 PP.addPPCallbacks( 699 collectPragmaMarksCallback(Clang->getSourceManager(), Marks)); 700 701 // FIXME: Attach a comment handler to take care of 702 // keep/export/no_include etc. IWYU pragmas. 703 704 // Collect tokens of the main file. 705 syntax::TokenCollector CollectTokens(PP); 706 707 // To remain consistent with preamble builds, these callbacks must be called 708 // exactly here, after preprocessor is initialized and BeginSourceFile() was 709 // called already. 710 for (const auto &L : ASTListeners) 711 L->beforeExecute(*Clang); 712 713 if (llvm::Error Err = Action->Execute()) 714 log("Execute() failed when building AST for {0}: {1}", MainInput.getFile(), 715 toString(std::move(Err))); 716 717 // Disable the macro collector for the remainder of this function, e.g. 718 // clang-tidy checkers. 719 MacroCollectorPtr->doneParse(); 720 721 // We have to consume the tokens before running clang-tidy to avoid collecting 722 // tokens from running the preprocessor inside the checks (only 723 // modernize-use-trailing-return-type does that today). 724 syntax::TokenBuffer Tokens = std::move(CollectTokens).consume(); 725 // Makes SelectionTree build much faster. 726 Tokens.indexExpandedTokens(); 727 std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls(); 728 // AST traversals should exclude the preamble, to avoid performance cliffs. 729 Clang->getASTContext().setTraversalScope(ParsedDecls); 730 if (!CTChecks.empty()) { 731 // Run the AST-dependent part of the clang-tidy checks. 732 // (The preprocessor part ran already, via PPCallbacks). 733 trace::Span Tracer("ClangTidyMatch"); 734 CTFinder.matchAST(Clang->getASTContext()); 735 } 736 737 // XXX: This is messy: clang-tidy checks flush some diagnostics at EOF. 738 // However Action->EndSourceFile() would destroy the ASTContext! 739 // So just inform the preprocessor of EOF, while keeping everything alive. 740 PP.EndSourceFile(); 741 // UnitDiagsConsumer is local, we can not store it in CompilerInstance that 742 // has a longer lifetime. 743 Clang->getDiagnostics().setClient(new IgnoreDiagnostics); 744 // CompilerInstance won't run this callback, do it directly. 745 ASTDiags.EndSourceFile(); 746 747 std::vector<Diag> Diags = CompilerInvocationDiags; 748 // FIXME: Also skip generation of diagnostics altogether to speed up ast 749 // builds when we are patching a stale preamble. 750 // Add diagnostics from the preamble, if any. 751 if (Preamble) 752 llvm::append_range(Diags, Patch->patchedDiags()); 753 // Finally, add diagnostics coming from the AST. 754 { 755 std::vector<Diag> D = ASTDiags.take(&*CTContext); 756 Diags.insert(Diags.end(), D.begin(), D.end()); 757 } 758 ParsedAST Result(Filename, Inputs.Version, std::move(Preamble), 759 std::move(Clang), std::move(Action), std::move(Tokens), 760 std::move(Macros), std::move(Marks), std::move(ParsedDecls), 761 std::move(Diags), std::move(Includes), std::move(PI)); 762 llvm::move(getIncludeCleanerDiags(Result, Inputs.Contents, *Inputs.TFS), 763 std::back_inserter(Result.Diags)); 764 return std::move(Result); 765 } 766 767 ParsedAST::ParsedAST(ParsedAST &&Other) = default; 768 769 ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default; 770 771 ParsedAST::~ParsedAST() { 772 if (Action) { 773 // We already notified the PP of end-of-file earlier, so detach it first. 774 // We must keep it alive until after EndSourceFile(), Sema relies on this. 775 auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now. 776 Clang->setPreprocessor(nullptr); // Detach so we don't send EOF again. 777 Action->EndSourceFile(); // Destroy ASTContext and Sema. 778 // Now Sema is gone, it's safe for PP to go out of scope. 779 } 780 } 781 782 ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); } 783 784 const ASTContext &ParsedAST::getASTContext() const { 785 return Clang->getASTContext(); 786 } 787 788 Sema &ParsedAST::getSema() { return Clang->getSema(); } 789 790 Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); } 791 792 std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() { 793 return Clang->getPreprocessorPtr(); 794 } 795 796 const Preprocessor &ParsedAST::getPreprocessor() const { 797 return Clang->getPreprocessor(); 798 } 799 800 llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() { 801 return LocalTopLevelDecls; 802 } 803 804 llvm::ArrayRef<const Decl *> ParsedAST::getLocalTopLevelDecls() const { 805 return LocalTopLevelDecls; 806 } 807 808 const MainFileMacros &ParsedAST::getMacros() const { return Macros; } 809 const std::vector<PragmaMark> &ParsedAST::getMarks() const { return Marks; } 810 811 std::size_t ParsedAST::getUsedBytes() const { 812 auto &AST = getASTContext(); 813 // FIXME(ibiryukov): we do not account for the dynamically allocated part of 814 // Message and Fixes inside each diagnostic. 815 std::size_t Total = 816 clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags); 817 818 // FIXME: the rest of the function is almost a direct copy-paste from 819 // libclang's clang_getCXTUResourceUsage. We could share the implementation. 820 821 // Sum up various allocators inside the ast context and the preprocessor. 822 Total += AST.getASTAllocatedMemory(); 823 Total += AST.getSideTableAllocatedMemory(); 824 Total += AST.Idents.getAllocator().getTotalMemory(); 825 Total += AST.Selectors.getTotalMemory(); 826 827 Total += AST.getSourceManager().getContentCacheSize(); 828 Total += AST.getSourceManager().getDataStructureSizes(); 829 Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes; 830 831 if (ExternalASTSource *Ext = AST.getExternalSource()) 832 Total += Ext->getMemoryBufferSizes().malloc_bytes; 833 834 const Preprocessor &PP = getPreprocessor(); 835 Total += PP.getTotalMemory(); 836 if (PreprocessingRecord *PRec = PP.getPreprocessingRecord()) 837 Total += PRec->getTotalMemory(); 838 Total += PP.getHeaderSearchInfo().getTotalMemory(); 839 840 return Total; 841 } 842 843 const IncludeStructure &ParsedAST::getIncludeStructure() const { 844 return Includes; 845 } 846 847 ParsedAST::ParsedAST(PathRef TUPath, llvm::StringRef Version, 848 std::shared_ptr<const PreambleData> Preamble, 849 std::unique_ptr<CompilerInstance> Clang, 850 std::unique_ptr<FrontendAction> Action, 851 syntax::TokenBuffer Tokens, MainFileMacros Macros, 852 std::vector<PragmaMark> Marks, 853 std::vector<Decl *> LocalTopLevelDecls, 854 std::vector<Diag> Diags, IncludeStructure Includes, 855 include_cleaner::PragmaIncludes PI) 856 : TUPath(TUPath), Version(Version), Preamble(std::move(Preamble)), 857 Clang(std::move(Clang)), Action(std::move(Action)), 858 Tokens(std::move(Tokens)), Macros(std::move(Macros)), 859 Marks(std::move(Marks)), Diags(std::move(Diags)), 860 LocalTopLevelDecls(std::move(LocalTopLevelDecls)), 861 Includes(std::move(Includes)), PI(std::move(PI)), 862 Resolver(std::make_unique<HeuristicResolver>(getASTContext())) { 863 assert(this->Clang); 864 assert(this->Action); 865 } 866 867 const include_cleaner::PragmaIncludes &ParsedAST::getPragmaIncludes() const { 868 return PI; 869 } 870 871 std::optional<llvm::StringRef> ParsedAST::preambleVersion() const { 872 if (!Preamble) 873 return std::nullopt; 874 return llvm::StringRef(Preamble->Version); 875 } 876 877 llvm::ArrayRef<Diag> ParsedAST::getDiagnostics() const { return Diags; } 878 } // namespace clangd 879 } // namespace clang 880