1 //===--- SystemIncludeExtractor.cpp ------------------------------*- C++-*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // Some compiler drivers have implicit search mechanism for system headers. 9 // This compilation database implementation tries to extract that information by 10 // executing the driver in verbose mode. gcc-compatible drivers print something 11 // like: 12 // .... 13 // .... 14 // #include <...> search starts here: 15 // /usr/lib/gcc/x86_64-linux-gnu/7/include 16 // /usr/local/include 17 // /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed 18 // /usr/include/x86_64-linux-gnu 19 // /usr/include 20 // End of search list. 21 // .... 22 // .... 23 // This component parses that output and adds each path to command line args 24 // provided by Base, after prepending them with -isystem. Therefore current 25 // implementation would not work with a driver that is not gcc-compatible. 26 // 27 // First argument of the command line received from underlying compilation 28 // database is used as compiler driver path. Due to this arbitrary binary 29 // execution, this mechanism is not used by default and only executes binaries 30 // in the paths that are explicitly included by the user. 31 32 #include "CompileCommands.h" 33 #include "GlobalCompilationDatabase.h" 34 #include "support/Logger.h" 35 #include "support/Threading.h" 36 #include "support/Trace.h" 37 #include "clang/Basic/Diagnostic.h" 38 #include "clang/Basic/DiagnosticIDs.h" 39 #include "clang/Basic/DiagnosticOptions.h" 40 #include "clang/Basic/TargetInfo.h" 41 #include "clang/Basic/TargetOptions.h" 42 #include "clang/Driver/Types.h" 43 #include "clang/Tooling/CompilationDatabase.h" 44 #include "llvm/ADT/ArrayRef.h" 45 #include "llvm/ADT/DenseMap.h" 46 #include "llvm/ADT/Hashing.h" 47 #include "llvm/ADT/IntrusiveRefCntPtr.h" 48 #include "llvm/ADT/STLExtras.h" 49 #include "llvm/ADT/ScopeExit.h" 50 #include "llvm/ADT/SmallString.h" 51 #include "llvm/ADT/SmallVector.h" 52 #include "llvm/ADT/StringExtras.h" 53 #include "llvm/ADT/StringRef.h" 54 #include "llvm/Support/ErrorHandling.h" 55 #include "llvm/Support/FileSystem.h" 56 #include "llvm/Support/MemoryBuffer.h" 57 #include "llvm/Support/Path.h" 58 #include "llvm/Support/Program.h" 59 #include "llvm/Support/Regex.h" 60 #include "llvm/Support/ScopedPrinter.h" 61 #include "llvm/Support/raw_ostream.h" 62 #include <cassert> 63 #include <cstddef> 64 #include <iterator> 65 #include <memory> 66 #include <optional> 67 #include <string> 68 #include <tuple> 69 #include <utility> 70 #include <vector> 71 72 namespace clang::clangd { 73 namespace { 74 75 struct DriverInfo { 76 std::vector<std::string> SystemIncludes; 77 std::string Target; 78 }; 79 80 struct DriverArgs { 81 // Name of the driver program to execute or absolute path to it. 82 std::string Driver; 83 // Whether certain includes should be part of query. 84 bool StandardIncludes = true; 85 bool StandardCXXIncludes = true; 86 // Language to use while querying. 87 std::string Lang; 88 std::string Sysroot; 89 std::string ISysroot; 90 std::string Target; 91 std::string Stdlib; 92 llvm::SmallVector<std::string> Specs; 93 94 bool operator==(const DriverArgs &RHS) const { 95 return std::tie(Driver, StandardIncludes, StandardCXXIncludes, Lang, 96 Sysroot, ISysroot, Target, Stdlib, Specs) == 97 std::tie(RHS.Driver, RHS.StandardIncludes, RHS.StandardCXXIncludes, 98 RHS.Lang, RHS.Sysroot, RHS.ISysroot, RHS.Target, RHS.Stdlib, 99 RHS.Specs); 100 } 101 102 DriverArgs(const tooling::CompileCommand &Cmd, llvm::StringRef File) { 103 llvm::SmallString<128> Driver(Cmd.CommandLine.front()); 104 // Driver is a not a single executable name but instead a path (either 105 // relative or absolute). 106 if (llvm::any_of(Driver, 107 [](char C) { return llvm::sys::path::is_separator(C); })) { 108 llvm::sys::fs::make_absolute(Cmd.Directory, Driver); 109 } 110 this->Driver = Driver.str().str(); 111 for (size_t I = 0, E = Cmd.CommandLine.size(); I < E; ++I) { 112 llvm::StringRef Arg = Cmd.CommandLine[I]; 113 114 // Look for Language related flags. 115 if (Arg.consume_front("-x")) { 116 if (Arg.empty() && I + 1 < E) 117 Lang = Cmd.CommandLine[I + 1]; 118 else 119 Lang = Arg.str(); 120 } 121 // Look for standard/builtin includes. 122 else if (Arg == "-nostdinc" || Arg == "--no-standard-includes") 123 StandardIncludes = false; 124 else if (Arg == "-nostdinc++") 125 StandardCXXIncludes = false; 126 // Figure out sysroot 127 else if (Arg.consume_front("--sysroot")) { 128 if (Arg.consume_front("=")) 129 Sysroot = Arg.str(); 130 else if (Arg.empty() && I + 1 < E) 131 Sysroot = Cmd.CommandLine[I + 1]; 132 } else if (Arg.consume_front("-isysroot")) { 133 if (Arg.empty() && I + 1 < E) 134 ISysroot = Cmd.CommandLine[I + 1]; 135 else 136 ISysroot = Arg.str(); 137 } else if (Arg.consume_front("--target=")) { 138 Target = Arg.str(); 139 } else if (Arg.consume_front("-target")) { 140 if (Arg.empty() && I + 1 < E) 141 Target = Cmd.CommandLine[I + 1]; 142 } else if (Arg.consume_front("--stdlib")) { 143 if (Arg.consume_front("=")) 144 Stdlib = Arg.str(); 145 else if (Arg.empty() && I + 1 < E) 146 Stdlib = Cmd.CommandLine[I + 1]; 147 } else if (Arg.consume_front("-stdlib=")) { 148 Stdlib = Arg.str(); 149 } else if (Arg.starts_with("-specs=")) { 150 // clang requires a single token like `-specs=file` or `--specs=file`, 151 // but gcc will accept two tokens like `--specs file`. Since the 152 // compilation database is presumably correct, we just forward the flags 153 // as-is. 154 Specs.push_back(Arg.str()); 155 } else if (Arg.starts_with("--specs=")) { 156 Specs.push_back(Arg.str()); 157 } else if (Arg == "--specs" && I + 1 < E) { 158 Specs.push_back(Arg.str()); 159 Specs.push_back(Cmd.CommandLine[I + 1]); 160 } 161 } 162 163 // Downgrade objective-c++-header (used in clangd's fallback flags for .h 164 // files) to c++-header, as some drivers may fail to run the extraction 165 // command if it contains `-xobjective-c++-header` and objective-c++ support 166 // is not installed. 167 // In practice, we don't see different include paths for the two on 168 // clang+mac, which is the most common objectve-c compiler. 169 if (Lang == "objective-c++-header") { 170 Lang = "c++-header"; 171 } 172 173 // If language is not explicit in the flags, infer from the file. 174 // This is important as we want to cache each language separately. 175 if (Lang.empty()) { 176 llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.'); 177 auto Type = driver::types::lookupTypeForExtension(Ext); 178 if (Type == driver::types::TY_INVALID) { 179 elog("System include extraction: invalid file type for {0}", Ext); 180 } else { 181 Lang = driver::types::getTypeName(Type); 182 } 183 } 184 } 185 llvm::SmallVector<llvm::StringRef> render() const { 186 // FIXME: Don't treat lang specially? 187 assert(!Lang.empty()); 188 llvm::SmallVector<llvm::StringRef> Args = {"-x", Lang}; 189 if (!StandardIncludes) 190 Args.push_back("-nostdinc"); 191 if (!StandardCXXIncludes) 192 Args.push_back("-nostdinc++"); 193 if (!Sysroot.empty()) 194 Args.append({"--sysroot", Sysroot}); 195 if (!ISysroot.empty()) 196 Args.append({"-isysroot", ISysroot}); 197 if (!Target.empty()) 198 Args.append({"-target", Target}); 199 if (!Stdlib.empty()) 200 Args.append({"--stdlib", Stdlib}); 201 202 for (llvm::StringRef Spec : Specs) { 203 Args.push_back(Spec); 204 } 205 206 return Args; 207 } 208 209 static DriverArgs getEmpty() { return {}; } 210 211 private: 212 DriverArgs() = default; 213 }; 214 } // namespace 215 } // namespace clang::clangd 216 namespace llvm { 217 using DriverArgs = clang::clangd::DriverArgs; 218 template <> struct DenseMapInfo<DriverArgs> { 219 static DriverArgs getEmptyKey() { 220 auto Driver = DriverArgs::getEmpty(); 221 Driver.Driver = "EMPTY_KEY"; 222 return Driver; 223 } 224 static DriverArgs getTombstoneKey() { 225 auto Driver = DriverArgs::getEmpty(); 226 Driver.Driver = "TOMBSTONE_KEY"; 227 return Driver; 228 } 229 static unsigned getHashValue(const DriverArgs &Val) { 230 unsigned FixedFieldsHash = llvm::hash_value(std::tuple{ 231 Val.Driver, 232 Val.StandardIncludes, 233 Val.StandardCXXIncludes, 234 Val.Lang, 235 Val.Sysroot, 236 Val.ISysroot, 237 Val.Target, 238 Val.Stdlib, 239 }); 240 241 unsigned SpecsHash = 242 llvm::hash_combine_range(Val.Specs.begin(), Val.Specs.end()); 243 244 return llvm::hash_combine(FixedFieldsHash, SpecsHash); 245 } 246 static bool isEqual(const DriverArgs &LHS, const DriverArgs &RHS) { 247 return LHS == RHS; 248 } 249 }; 250 } // namespace llvm 251 namespace clang::clangd { 252 namespace { 253 bool isValidTarget(llvm::StringRef Triple) { 254 std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions); 255 TargetOpts->Triple = Triple.str(); 256 DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions, 257 new IgnoringDiagConsumer); 258 llvm::IntrusiveRefCntPtr<TargetInfo> Target = 259 TargetInfo::CreateTargetInfo(Diags, TargetOpts); 260 return bool(Target); 261 } 262 263 std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) { 264 DriverInfo Info; 265 const char SIS[] = "#include <...> search starts here:"; 266 const char SIE[] = "End of search list."; 267 const char TS[] = "Target: "; 268 llvm::SmallVector<llvm::StringRef> Lines; 269 Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); 270 271 enum { 272 Initial, // Initial state: searching for target or includes list. 273 IncludesExtracting, // Includes extracting. 274 Done // Includes and target extraction done. 275 } State = Initial; 276 bool SeenIncludes = false; 277 bool SeenTarget = false; 278 for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) { 279 auto Line = *It; 280 switch (State) { 281 case Initial: 282 if (!SeenIncludes && Line.trim() == SIS) { 283 SeenIncludes = true; 284 State = IncludesExtracting; 285 } else if (!SeenTarget && Line.trim().starts_with(TS)) { 286 SeenTarget = true; 287 llvm::StringRef TargetLine = Line.trim(); 288 TargetLine.consume_front(TS); 289 // Only detect targets that clang understands 290 if (!isValidTarget(TargetLine)) { 291 elog("System include extraction: invalid target \"{0}\", ignoring", 292 TargetLine); 293 } else { 294 Info.Target = TargetLine.str(); 295 vlog("System include extraction: target extracted: \"{0}\"", 296 TargetLine); 297 } 298 } 299 break; 300 case IncludesExtracting: 301 if (Line.trim() == SIE) { 302 State = SeenTarget ? Done : Initial; 303 } else { 304 Info.SystemIncludes.push_back(Line.trim().str()); 305 vlog("System include extraction: adding {0}", Line); 306 } 307 break; 308 default: 309 llvm_unreachable("Impossible state of the driver output parser"); 310 break; 311 } 312 } 313 if (!SeenIncludes) { 314 elog("System include extraction: start marker not found: {0}", Output); 315 return std::nullopt; 316 } 317 if (State == IncludesExtracting) { 318 elog("System include extraction: end marker missing: {0}", Output); 319 return std::nullopt; 320 } 321 return std::move(Info); 322 } 323 324 std::optional<std::string> run(llvm::ArrayRef<llvm::StringRef> Argv, 325 bool OutputIsStderr) { 326 llvm::SmallString<128> OutputPath; 327 if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd", 328 OutputPath)) { 329 elog("System include extraction: failed to create temporary file with " 330 "error {0}", 331 EC.message()); 332 return std::nullopt; 333 } 334 auto CleanUp = llvm::make_scope_exit( 335 [&OutputPath]() { llvm::sys::fs::remove(OutputPath); }); 336 337 std::optional<llvm::StringRef> Redirects[] = {{""}, {""}, {""}}; 338 Redirects[OutputIsStderr ? 2 : 1] = OutputPath.str(); 339 340 std::string ErrMsg; 341 if (int RC = 342 llvm::sys::ExecuteAndWait(Argv.front(), Argv, /*Env=*/std::nullopt, 343 Redirects, /*SecondsToWait=*/0, 344 /*MemoryLimit=*/0, &ErrMsg)) { 345 elog("System include extraction: driver execution failed with return code: " 346 "{0} - '{1}'. Args: [{2}]", 347 llvm::to_string(RC), ErrMsg, printArgv(Argv)); 348 return std::nullopt; 349 } 350 351 auto BufOrError = llvm::MemoryBuffer::getFile(OutputPath); 352 if (!BufOrError) { 353 elog("System include extraction: failed to read {0} with error {1}", 354 OutputPath, BufOrError.getError().message()); 355 return std::nullopt; 356 } 357 return BufOrError.get().get()->getBuffer().str(); 358 } 359 360 std::optional<DriverInfo> 361 extractSystemIncludesAndTarget(const DriverArgs &InputArgs, 362 const llvm::Regex &QueryDriverRegex) { 363 trace::Span Tracer("Extract system includes and target"); 364 365 std::string Driver = InputArgs.Driver; 366 if (!llvm::sys::path::is_absolute(Driver)) { 367 auto DriverProgram = llvm::sys::findProgramByName(Driver); 368 if (DriverProgram) { 369 vlog("System include extraction: driver {0} expanded to {1}", Driver, 370 *DriverProgram); 371 Driver = *DriverProgram; 372 } else { 373 elog("System include extraction: driver {0} not found in PATH", Driver); 374 return std::nullopt; 375 } 376 } 377 378 SPAN_ATTACH(Tracer, "driver", Driver); 379 SPAN_ATTACH(Tracer, "lang", InputArgs.Lang); 380 381 // If driver was "../foo" then having to allowlist "/path/a/../foo" rather 382 // than "/path/foo" is absurd. 383 // Allow either to match the allowlist, then proceed with "/path/a/../foo". 384 // This was our historical behavior, and it *could* resolve to something else. 385 llvm::SmallString<256> NoDots(Driver); 386 llvm::sys::path::remove_dots(NoDots, /*remove_dot_dot=*/true); 387 if (!QueryDriverRegex.match(Driver) && !QueryDriverRegex.match(NoDots)) { 388 vlog("System include extraction: not allowed driver {0}", Driver); 389 return std::nullopt; 390 } 391 392 llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-v"}; 393 Args.append(InputArgs.render()); 394 // Input needs to go after Lang flags. 395 Args.push_back("-"); 396 auto Output = run(Args, /*OutputIsStderr=*/true); 397 if (!Output) 398 return std::nullopt; 399 400 std::optional<DriverInfo> Info = parseDriverOutput(*Output); 401 if (!Info) 402 return std::nullopt; 403 404 // The built-in headers are tightly coupled to parser builtins. 405 // (These are clang's "resource dir", GCC's GCC_INCLUDE_DIR.) 406 // We should keep using clangd's versions, so exclude the queried builtins. 407 // They're not specially marked in the -v output, but we can get the path 408 // with `$DRIVER -print-file-name=include`. 409 if (auto BuiltinHeaders = 410 run({Driver, "-print-file-name=include"}, /*OutputIsStderr=*/false)) { 411 auto Path = llvm::StringRef(*BuiltinHeaders).trim(); 412 if (!Path.empty() && llvm::sys::path::is_absolute(Path)) { 413 auto Size = Info->SystemIncludes.size(); 414 llvm::erase(Info->SystemIncludes, Path); 415 vlog("System includes extractor: builtin headers {0} {1}", Path, 416 (Info->SystemIncludes.size() != Size) 417 ? "excluded" 418 : "not found in driver's response"); 419 } 420 } 421 422 log("System includes extractor: successfully executed {0}\n\tgot includes: " 423 "\"{1}\"\n\tgot target: \"{2}\"", 424 Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target); 425 return Info; 426 } 427 428 tooling::CompileCommand & 429 addSystemIncludes(tooling::CompileCommand &Cmd, 430 llvm::ArrayRef<std::string> SystemIncludes) { 431 std::vector<std::string> ToAppend; 432 for (llvm::StringRef Include : SystemIncludes) { 433 // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl" 434 ToAppend.push_back("-isystem"); 435 ToAppend.push_back(Include.str()); 436 } 437 if (!ToAppend.empty()) { 438 // Just append when `--` isn't present. 439 auto InsertAt = llvm::find(Cmd.CommandLine, "--"); 440 Cmd.CommandLine.insert(InsertAt, std::make_move_iterator(ToAppend.begin()), 441 std::make_move_iterator(ToAppend.end())); 442 } 443 return Cmd; 444 } 445 446 tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd, 447 const std::string &Target) { 448 if (!Target.empty()) { 449 // We do not want to override existing target with extracted one. 450 for (llvm::StringRef Arg : Cmd.CommandLine) { 451 if (Arg == "-target" || Arg.starts_with("--target=")) 452 return Cmd; 453 } 454 // Just append when `--` isn't present. 455 auto InsertAt = llvm::find(Cmd.CommandLine, "--"); 456 Cmd.CommandLine.insert(InsertAt, "--target=" + Target); 457 } 458 return Cmd; 459 } 460 461 /// Converts a glob containing only ** or * into a regex. 462 std::string convertGlobToRegex(llvm::StringRef Glob) { 463 std::string RegText; 464 llvm::raw_string_ostream RegStream(RegText); 465 RegStream << '^'; 466 for (size_t I = 0, E = Glob.size(); I < E; ++I) { 467 if (Glob[I] == '*') { 468 if (I + 1 < E && Glob[I + 1] == '*') { 469 // Double star, accept any sequence. 470 RegStream << ".*"; 471 // Also skip the second star. 472 ++I; 473 } else { 474 // Single star, accept any sequence without a slash. 475 RegStream << "[^/]*"; 476 } 477 } else if (llvm::sys::path::is_separator(Glob[I]) && 478 llvm::sys::path::is_separator('/') && 479 llvm::sys::path::is_separator('\\')) { 480 RegStream << R"([/\\])"; // Accept either slash on windows. 481 } else { 482 RegStream << llvm::Regex::escape(Glob.substr(I, 1)); 483 } 484 } 485 RegStream << '$'; 486 return RegText; 487 } 488 489 /// Converts a glob containing only ** or * into a regex. 490 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) { 491 assert(!Globs.empty() && "Globs cannot be empty!"); 492 std::vector<std::string> RegTexts; 493 RegTexts.reserve(Globs.size()); 494 for (llvm::StringRef Glob : Globs) 495 RegTexts.push_back(convertGlobToRegex(Glob)); 496 497 // Tempting to pass IgnoreCase, but we don't know the FS sensitivity. 498 llvm::Regex Reg(llvm::join(RegTexts, "|")); 499 assert(Reg.isValid(RegTexts.front()) && 500 "Created an invalid regex from globs"); 501 return Reg; 502 } 503 504 /// Extracts system includes from a trusted driver by parsing the output of 505 /// include search path and appends them to the commands coming from underlying 506 /// compilation database. 507 class SystemIncludeExtractor { 508 public: 509 SystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) 510 : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {} 511 512 void operator()(tooling::CompileCommand &Cmd, llvm::StringRef File) const { 513 if (Cmd.CommandLine.empty()) 514 return; 515 516 DriverArgs Args(Cmd, File); 517 if (Args.Lang.empty()) 518 return; 519 if (auto Info = QueriedDrivers.get(Args, [&] { 520 return extractSystemIncludesAndTarget(Args, QueryDriverRegex); 521 })) { 522 setTarget(addSystemIncludes(Cmd, Info->SystemIncludes), Info->Target); 523 } 524 } 525 526 private: 527 // Caches includes extracted from a driver. Key is driver:lang. 528 Memoize<llvm::DenseMap<DriverArgs, std::optional<DriverInfo>>> QueriedDrivers; 529 llvm::Regex QueryDriverRegex; 530 }; 531 } // namespace 532 533 SystemIncludeExtractorFn 534 getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) { 535 if (QueryDriverGlobs.empty()) 536 return nullptr; 537 return SystemIncludeExtractor(QueryDriverGlobs); 538 } 539 540 } // namespace clang::clangd 541