10eae32dcSDimitry Andric //===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===// 20eae32dcSDimitry Andric // 30eae32dcSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40eae32dcSDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50eae32dcSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60eae32dcSDimitry Andric // 70eae32dcSDimitry Andric //===----------------------------------------------------------------------===// 80eae32dcSDimitry Andric /// 90eae32dcSDimitry Andric /// \file 100eae32dcSDimitry Andric /// 11753f127fSDimitry Andric /// This file contains several definitions for the debuginfod client and server. 12753f127fSDimitry Andric /// For the client, this file defines the fetchInfo function. For the server, 13753f127fSDimitry Andric /// this file defines the DebuginfodLogEntry and DebuginfodServer structs, as 14753f127fSDimitry Andric /// well as the DebuginfodLog, DebuginfodCollection classes. The fetchInfo 15753f127fSDimitry Andric /// function retrieves any of the three supported artifact types: (executable, 16753f127fSDimitry Andric /// debuginfo, source file) associated with a build-id from debuginfod servers. 17753f127fSDimitry Andric /// If a source file is to be fetched, its absolute path must be specified in 18753f127fSDimitry Andric /// the Description argument to fetchInfo. The DebuginfodLogEntry, 19753f127fSDimitry Andric /// DebuginfodLog, and DebuginfodCollection are used by the DebuginfodServer to 20753f127fSDimitry Andric /// scan the local filesystem for binaries and serve the debuginfod protocol. 210eae32dcSDimitry Andric /// 220eae32dcSDimitry Andric //===----------------------------------------------------------------------===// 230eae32dcSDimitry Andric 240eae32dcSDimitry Andric #include "llvm/Debuginfod/Debuginfod.h" 25bdd1243dSDimitry Andric #include "llvm/ADT/StringExtras.h" 260eae32dcSDimitry Andric #include "llvm/ADT/StringRef.h" 27753f127fSDimitry Andric #include "llvm/BinaryFormat/Magic.h" 28753f127fSDimitry Andric #include "llvm/DebugInfo/DWARF/DWARFContext.h" 29753f127fSDimitry Andric #include "llvm/DebugInfo/Symbolize/Symbolize.h" 300eae32dcSDimitry Andric #include "llvm/Debuginfod/HTTPClient.h" 31bdd1243dSDimitry Andric #include "llvm/Object/BuildID.h" 32753f127fSDimitry Andric #include "llvm/Object/ELFObjectFile.h" 330eae32dcSDimitry Andric #include "llvm/Support/CachePruning.h" 340eae32dcSDimitry Andric #include "llvm/Support/Caching.h" 3504eeddc0SDimitry Andric #include "llvm/Support/Errc.h" 360eae32dcSDimitry Andric #include "llvm/Support/Error.h" 370eae32dcSDimitry Andric #include "llvm/Support/FileUtilities.h" 38bdd1243dSDimitry Andric #include "llvm/Support/MemoryBuffer.h" 3904eeddc0SDimitry Andric #include "llvm/Support/Path.h" 40753f127fSDimitry Andric #include "llvm/Support/ThreadPool.h" 410eae32dcSDimitry Andric #include "llvm/Support/xxhash.h" 420eae32dcSDimitry Andric 43753f127fSDimitry Andric #include <atomic> 445f757f3fSDimitry Andric #include <optional> 45bdd1243dSDimitry Andric #include <thread> 46753f127fSDimitry Andric 470eae32dcSDimitry Andric namespace llvm { 48bdd1243dSDimitry Andric 49bdd1243dSDimitry Andric using llvm::object::BuildIDRef; 50bdd1243dSDimitry Andric 515f757f3fSDimitry Andric namespace { 525f757f3fSDimitry Andric std::optional<SmallVector<StringRef>> DebuginfodUrls; 535f757f3fSDimitry Andric // Many Readers/Single Writer lock protecting the global debuginfod URL list. 545f757f3fSDimitry Andric llvm::sys::RWMutex UrlsMutex; 555f757f3fSDimitry Andric } // namespace 565f757f3fSDimitry Andric 577a6dacacSDimitry Andric std::string getDebuginfodCacheKey(llvm::StringRef S) { 585f757f3fSDimitry Andric return utostr(xxh3_64bits(S)); 595f757f3fSDimitry Andric } 600eae32dcSDimitry Andric 610eae32dcSDimitry Andric // Returns a binary BuildID as a normalized hex string. 620eae32dcSDimitry Andric // Uses lowercase for compatibility with common debuginfod servers. 630eae32dcSDimitry Andric static std::string buildIDToString(BuildIDRef ID) { 640eae32dcSDimitry Andric return llvm::toHex(ID, /*LowerCase=*/true); 650eae32dcSDimitry Andric } 660eae32dcSDimitry Andric 671ac55f4cSDimitry Andric bool canUseDebuginfod() { 681ac55f4cSDimitry Andric return HTTPClient::isAvailable() && !getDefaultDebuginfodUrls().empty(); 691ac55f4cSDimitry Andric } 701ac55f4cSDimitry Andric 711ac55f4cSDimitry Andric SmallVector<StringRef> getDefaultDebuginfodUrls() { 725f757f3fSDimitry Andric std::shared_lock<llvm::sys::RWMutex> ReadGuard(UrlsMutex); 735f757f3fSDimitry Andric if (!DebuginfodUrls) { 745f757f3fSDimitry Andric // Only read from the environment variable if the user hasn't already 757a6dacacSDimitry Andric // set the value. 765f757f3fSDimitry Andric ReadGuard.unlock(); 775f757f3fSDimitry Andric std::unique_lock<llvm::sys::RWMutex> WriteGuard(UrlsMutex); 785f757f3fSDimitry Andric DebuginfodUrls = SmallVector<StringRef>(); 795f757f3fSDimitry Andric if (const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS")) { 805f757f3fSDimitry Andric StringRef(DebuginfodUrlsEnv) 815f757f3fSDimitry Andric .split(DebuginfodUrls.value(), " ", -1, false); 825f757f3fSDimitry Andric } 835f757f3fSDimitry Andric WriteGuard.unlock(); 845f757f3fSDimitry Andric ReadGuard.lock(); 855f757f3fSDimitry Andric } 865f757f3fSDimitry Andric return DebuginfodUrls.value(); 875f757f3fSDimitry Andric } 880eae32dcSDimitry Andric 897a6dacacSDimitry Andric // Set the default debuginfod URL list, override the environment variable. 905f757f3fSDimitry Andric void setDefaultDebuginfodUrls(const SmallVector<StringRef> &URLs) { 915f757f3fSDimitry Andric std::unique_lock<llvm::sys::RWMutex> WriteGuard(UrlsMutex); 925f757f3fSDimitry Andric DebuginfodUrls = URLs; 930eae32dcSDimitry Andric } 940eae32dcSDimitry Andric 95753f127fSDimitry Andric /// Finds a default local file caching directory for the debuginfod client, 96753f127fSDimitry Andric /// first checking DEBUGINFOD_CACHE_PATH. 970eae32dcSDimitry Andric Expected<std::string> getDefaultDebuginfodCacheDirectory() { 980eae32dcSDimitry Andric if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH")) 990eae32dcSDimitry Andric return CacheDirectoryEnv; 1000eae32dcSDimitry Andric 1010eae32dcSDimitry Andric SmallString<64> CacheDirectory; 1020eae32dcSDimitry Andric if (!sys::path::cache_directory(CacheDirectory)) 1030eae32dcSDimitry Andric return createStringError( 1040eae32dcSDimitry Andric errc::io_error, "Unable to determine appropriate cache directory."); 10504eeddc0SDimitry Andric sys::path::append(CacheDirectory, "llvm-debuginfod", "client"); 1060eae32dcSDimitry Andric return std::string(CacheDirectory); 1070eae32dcSDimitry Andric } 1080eae32dcSDimitry Andric 1090eae32dcSDimitry Andric std::chrono::milliseconds getDefaultDebuginfodTimeout() { 1100eae32dcSDimitry Andric long Timeout; 1110eae32dcSDimitry Andric const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT"); 1120eae32dcSDimitry Andric if (DebuginfodTimeoutEnv && 1130eae32dcSDimitry Andric to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10)) 1140eae32dcSDimitry Andric return std::chrono::milliseconds(Timeout * 1000); 1150eae32dcSDimitry Andric 1160eae32dcSDimitry Andric return std::chrono::milliseconds(90 * 1000); 1170eae32dcSDimitry Andric } 1180eae32dcSDimitry Andric 1190eae32dcSDimitry Andric /// The following functions fetch a debuginfod artifact to a file in a local 1200eae32dcSDimitry Andric /// cache and return the cached file path. They first search the local cache, 1210eae32dcSDimitry Andric /// followed by the debuginfod servers. 1220eae32dcSDimitry Andric 1237a6dacacSDimitry Andric std::string getDebuginfodSourceUrlPath(BuildIDRef ID, 1240eae32dcSDimitry Andric StringRef SourceFilePath) { 1250eae32dcSDimitry Andric SmallString<64> UrlPath; 1260eae32dcSDimitry Andric sys::path::append(UrlPath, sys::path::Style::posix, "buildid", 1270eae32dcSDimitry Andric buildIDToString(ID), "source", 1280eae32dcSDimitry Andric sys::path::convert_to_slash(SourceFilePath)); 1297a6dacacSDimitry Andric return std::string(UrlPath); 1300eae32dcSDimitry Andric } 1310eae32dcSDimitry Andric 1327a6dacacSDimitry Andric Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID, 1337a6dacacSDimitry Andric StringRef SourceFilePath) { 1347a6dacacSDimitry Andric std::string UrlPath = getDebuginfodSourceUrlPath(ID, SourceFilePath); 1357a6dacacSDimitry Andric return getCachedOrDownloadArtifact(getDebuginfodCacheKey(UrlPath), UrlPath); 1367a6dacacSDimitry Andric } 1377a6dacacSDimitry Andric 1387a6dacacSDimitry Andric std::string getDebuginfodExecutableUrlPath(BuildIDRef ID) { 1390eae32dcSDimitry Andric SmallString<64> UrlPath; 1400eae32dcSDimitry Andric sys::path::append(UrlPath, sys::path::Style::posix, "buildid", 1410eae32dcSDimitry Andric buildIDToString(ID), "executable"); 1427a6dacacSDimitry Andric return std::string(UrlPath); 1430eae32dcSDimitry Andric } 1440eae32dcSDimitry Andric 1457a6dacacSDimitry Andric Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) { 1467a6dacacSDimitry Andric std::string UrlPath = getDebuginfodExecutableUrlPath(ID); 1477a6dacacSDimitry Andric return getCachedOrDownloadArtifact(getDebuginfodCacheKey(UrlPath), UrlPath); 1487a6dacacSDimitry Andric } 1497a6dacacSDimitry Andric 1507a6dacacSDimitry Andric std::string getDebuginfodDebuginfoUrlPath(BuildIDRef ID) { 1510eae32dcSDimitry Andric SmallString<64> UrlPath; 1520eae32dcSDimitry Andric sys::path::append(UrlPath, sys::path::Style::posix, "buildid", 1530eae32dcSDimitry Andric buildIDToString(ID), "debuginfo"); 1547a6dacacSDimitry Andric return std::string(UrlPath); 1557a6dacacSDimitry Andric } 1567a6dacacSDimitry Andric 1577a6dacacSDimitry Andric Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) { 1587a6dacacSDimitry Andric std::string UrlPath = getDebuginfodDebuginfoUrlPath(ID); 1597a6dacacSDimitry Andric return getCachedOrDownloadArtifact(getDebuginfodCacheKey(UrlPath), UrlPath); 1600eae32dcSDimitry Andric } 1610eae32dcSDimitry Andric 1620eae32dcSDimitry Andric // General fetching function. 1630eae32dcSDimitry Andric Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey, 1640eae32dcSDimitry Andric StringRef UrlPath) { 1650eae32dcSDimitry Andric SmallString<10> CacheDir; 1660eae32dcSDimitry Andric 1670eae32dcSDimitry Andric Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory(); 1680eae32dcSDimitry Andric if (!CacheDirOrErr) 1690eae32dcSDimitry Andric return CacheDirOrErr.takeError(); 1700eae32dcSDimitry Andric CacheDir = *CacheDirOrErr; 1710eae32dcSDimitry Andric 1720eae32dcSDimitry Andric return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir, 1731ac55f4cSDimitry Andric getDefaultDebuginfodUrls(), 1740eae32dcSDimitry Andric getDefaultDebuginfodTimeout()); 1750eae32dcSDimitry Andric } 1760eae32dcSDimitry Andric 17781ad6265SDimitry Andric namespace { 17881ad6265SDimitry Andric 17981ad6265SDimitry Andric /// A simple handler which streams the returned data to a cache file. The cache 18081ad6265SDimitry Andric /// file is only created if a 200 OK status is observed. 18181ad6265SDimitry Andric class StreamedHTTPResponseHandler : public HTTPResponseHandler { 18281ad6265SDimitry Andric using CreateStreamFn = 18381ad6265SDimitry Andric std::function<Expected<std::unique_ptr<CachedFileStream>>()>; 18481ad6265SDimitry Andric CreateStreamFn CreateStream; 18581ad6265SDimitry Andric HTTPClient &Client; 18681ad6265SDimitry Andric std::unique_ptr<CachedFileStream> FileStream; 18781ad6265SDimitry Andric 18881ad6265SDimitry Andric public: 18981ad6265SDimitry Andric StreamedHTTPResponseHandler(CreateStreamFn CreateStream, HTTPClient &Client) 19081ad6265SDimitry Andric : CreateStream(CreateStream), Client(Client) {} 19181ad6265SDimitry Andric virtual ~StreamedHTTPResponseHandler() = default; 19281ad6265SDimitry Andric 19381ad6265SDimitry Andric Error handleBodyChunk(StringRef BodyChunk) override; 19481ad6265SDimitry Andric }; 19581ad6265SDimitry Andric 19681ad6265SDimitry Andric } // namespace 19781ad6265SDimitry Andric 19881ad6265SDimitry Andric Error StreamedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) { 19981ad6265SDimitry Andric if (!FileStream) { 2001ac55f4cSDimitry Andric unsigned Code = Client.responseCode(); 2011ac55f4cSDimitry Andric if (Code && Code != 200) 20281ad6265SDimitry Andric return Error::success(); 20381ad6265SDimitry Andric Expected<std::unique_ptr<CachedFileStream>> FileStreamOrError = 20481ad6265SDimitry Andric CreateStream(); 20581ad6265SDimitry Andric if (!FileStreamOrError) 20681ad6265SDimitry Andric return FileStreamOrError.takeError(); 20781ad6265SDimitry Andric FileStream = std::move(*FileStreamOrError); 20881ad6265SDimitry Andric } 20981ad6265SDimitry Andric *FileStream->OS << BodyChunk; 21081ad6265SDimitry Andric return Error::success(); 21181ad6265SDimitry Andric } 21281ad6265SDimitry Andric 213bdd1243dSDimitry Andric // An over-accepting simplification of the HTTP RFC 7230 spec. 214bdd1243dSDimitry Andric static bool isHeader(StringRef S) { 215bdd1243dSDimitry Andric StringRef Name; 216bdd1243dSDimitry Andric StringRef Value; 217bdd1243dSDimitry Andric std::tie(Name, Value) = S.split(':'); 218bdd1243dSDimitry Andric if (Name.empty() || Value.empty()) 219bdd1243dSDimitry Andric return false; 220bdd1243dSDimitry Andric return all_of(Name, [](char C) { return llvm::isPrint(C) && C != ' '; }) && 221bdd1243dSDimitry Andric all_of(Value, [](char C) { return llvm::isPrint(C) || C == '\t'; }); 222bdd1243dSDimitry Andric } 223bdd1243dSDimitry Andric 224bdd1243dSDimitry Andric static SmallVector<std::string, 0> getHeaders() { 225bdd1243dSDimitry Andric const char *Filename = getenv("DEBUGINFOD_HEADERS_FILE"); 226bdd1243dSDimitry Andric if (!Filename) 227bdd1243dSDimitry Andric return {}; 228bdd1243dSDimitry Andric ErrorOr<std::unique_ptr<MemoryBuffer>> HeadersFile = 229bdd1243dSDimitry Andric MemoryBuffer::getFile(Filename, /*IsText=*/true); 230bdd1243dSDimitry Andric if (!HeadersFile) 231bdd1243dSDimitry Andric return {}; 232bdd1243dSDimitry Andric 233bdd1243dSDimitry Andric SmallVector<std::string, 0> Headers; 234bdd1243dSDimitry Andric uint64_t LineNumber = 0; 235bdd1243dSDimitry Andric for (StringRef Line : llvm::split((*HeadersFile)->getBuffer(), '\n')) { 236bdd1243dSDimitry Andric LineNumber++; 237bdd1243dSDimitry Andric if (!Line.empty() && Line.back() == '\r') 238bdd1243dSDimitry Andric Line = Line.drop_back(); 239bdd1243dSDimitry Andric if (!isHeader(Line)) { 240bdd1243dSDimitry Andric if (!all_of(Line, llvm::isSpace)) 241bdd1243dSDimitry Andric WithColor::warning() 242bdd1243dSDimitry Andric << "could not parse debuginfod header: " << Filename << ':' 243bdd1243dSDimitry Andric << LineNumber << '\n'; 244bdd1243dSDimitry Andric continue; 245bdd1243dSDimitry Andric } 246bdd1243dSDimitry Andric Headers.emplace_back(Line); 247bdd1243dSDimitry Andric } 248bdd1243dSDimitry Andric return Headers; 249bdd1243dSDimitry Andric } 250bdd1243dSDimitry Andric 2510eae32dcSDimitry Andric Expected<std::string> getCachedOrDownloadArtifact( 2520eae32dcSDimitry Andric StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath, 2530eae32dcSDimitry Andric ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) { 2540eae32dcSDimitry Andric SmallString<64> AbsCachedArtifactPath; 2550eae32dcSDimitry Andric sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath, 2560eae32dcSDimitry Andric "llvmcache-" + UniqueKey); 2570eae32dcSDimitry Andric 2580eae32dcSDimitry Andric Expected<FileCache> CacheOrErr = 2590eae32dcSDimitry Andric localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath); 2600eae32dcSDimitry Andric if (!CacheOrErr) 2610eae32dcSDimitry Andric return CacheOrErr.takeError(); 2620eae32dcSDimitry Andric 2630eae32dcSDimitry Andric FileCache Cache = *CacheOrErr; 2640eae32dcSDimitry Andric // We choose an arbitrary Task parameter as we do not make use of it. 2650eae32dcSDimitry Andric unsigned Task = 0; 266bdd1243dSDimitry Andric Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey, ""); 2670eae32dcSDimitry Andric if (!CacheAddStreamOrErr) 2680eae32dcSDimitry Andric return CacheAddStreamOrErr.takeError(); 2690eae32dcSDimitry Andric AddStreamFn &CacheAddStream = *CacheAddStreamOrErr; 2700eae32dcSDimitry Andric if (!CacheAddStream) 2710eae32dcSDimitry Andric return std::string(AbsCachedArtifactPath); 2720eae32dcSDimitry Andric // The artifact was not found in the local cache, query the debuginfod 2730eae32dcSDimitry Andric // servers. 2740eae32dcSDimitry Andric if (!HTTPClient::isAvailable()) 2750eae32dcSDimitry Andric return createStringError(errc::io_error, 2760eae32dcSDimitry Andric "No working HTTP client is available."); 2770eae32dcSDimitry Andric 2780eae32dcSDimitry Andric if (!HTTPClient::IsInitialized) 2790eae32dcSDimitry Andric return createStringError( 2800eae32dcSDimitry Andric errc::io_error, 2810eae32dcSDimitry Andric "A working HTTP client is available, but it is not initialized. To " 2820eae32dcSDimitry Andric "allow Debuginfod to make HTTP requests, call HTTPClient::initialize() " 2830eae32dcSDimitry Andric "at the beginning of main."); 2840eae32dcSDimitry Andric 2850eae32dcSDimitry Andric HTTPClient Client; 2860eae32dcSDimitry Andric Client.setTimeout(Timeout); 2870eae32dcSDimitry Andric for (StringRef ServerUrl : DebuginfodUrls) { 2880eae32dcSDimitry Andric SmallString<64> ArtifactUrl; 2890eae32dcSDimitry Andric sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath); 2900eae32dcSDimitry Andric 29181ad6265SDimitry Andric // Perform the HTTP request and if successful, write the response body to 29281ad6265SDimitry Andric // the cache. 29306c3fb27SDimitry Andric { 294bdd1243dSDimitry Andric StreamedHTTPResponseHandler Handler( 295bdd1243dSDimitry Andric [&]() { return CacheAddStream(Task, ""); }, Client); 29681ad6265SDimitry Andric HTTPRequest Request(ArtifactUrl); 297bdd1243dSDimitry Andric Request.Headers = getHeaders(); 29881ad6265SDimitry Andric Error Err = Client.perform(Request, Handler); 29981ad6265SDimitry Andric if (Err) 30081ad6265SDimitry Andric return std::move(Err); 3010eae32dcSDimitry Andric 3021ac55f4cSDimitry Andric unsigned Code = Client.responseCode(); 3031ac55f4cSDimitry Andric if (Code && Code != 200) 3040eae32dcSDimitry Andric continue; 30506c3fb27SDimitry Andric } 30606c3fb27SDimitry Andric 30706c3fb27SDimitry Andric Expected<CachePruningPolicy> PruningPolicyOrErr = 30806c3fb27SDimitry Andric parseCachePruningPolicy(std::getenv("DEBUGINFOD_CACHE_POLICY")); 30906c3fb27SDimitry Andric if (!PruningPolicyOrErr) 31006c3fb27SDimitry Andric return PruningPolicyOrErr.takeError(); 31106c3fb27SDimitry Andric pruneCache(CacheDirectoryPath, *PruningPolicyOrErr); 3120eae32dcSDimitry Andric 3130eae32dcSDimitry Andric // Return the path to the artifact on disk. 3140eae32dcSDimitry Andric return std::string(AbsCachedArtifactPath); 3150eae32dcSDimitry Andric } 3160eae32dcSDimitry Andric 3170eae32dcSDimitry Andric return createStringError(errc::argument_out_of_domain, "build id not found"); 3180eae32dcSDimitry Andric } 319753f127fSDimitry Andric 320753f127fSDimitry Andric DebuginfodLogEntry::DebuginfodLogEntry(const Twine &Message) 321753f127fSDimitry Andric : Message(Message.str()) {} 322753f127fSDimitry Andric 323753f127fSDimitry Andric void DebuginfodLog::push(const Twine &Message) { 324753f127fSDimitry Andric push(DebuginfodLogEntry(Message)); 325753f127fSDimitry Andric } 326753f127fSDimitry Andric 327753f127fSDimitry Andric void DebuginfodLog::push(DebuginfodLogEntry Entry) { 328753f127fSDimitry Andric { 329753f127fSDimitry Andric std::lock_guard<std::mutex> Guard(QueueMutex); 330753f127fSDimitry Andric LogEntryQueue.push(Entry); 331753f127fSDimitry Andric } 332753f127fSDimitry Andric QueueCondition.notify_one(); 333753f127fSDimitry Andric } 334753f127fSDimitry Andric 335753f127fSDimitry Andric DebuginfodLogEntry DebuginfodLog::pop() { 336753f127fSDimitry Andric { 337753f127fSDimitry Andric std::unique_lock<std::mutex> Guard(QueueMutex); 338753f127fSDimitry Andric // Wait for messages to be pushed into the queue. 339753f127fSDimitry Andric QueueCondition.wait(Guard, [&] { return !LogEntryQueue.empty(); }); 340753f127fSDimitry Andric } 341753f127fSDimitry Andric std::lock_guard<std::mutex> Guard(QueueMutex); 342753f127fSDimitry Andric if (!LogEntryQueue.size()) 343753f127fSDimitry Andric llvm_unreachable("Expected message in the queue."); 344753f127fSDimitry Andric 345753f127fSDimitry Andric DebuginfodLogEntry Entry = LogEntryQueue.front(); 346753f127fSDimitry Andric LogEntryQueue.pop(); 347753f127fSDimitry Andric return Entry; 348753f127fSDimitry Andric } 349753f127fSDimitry Andric 350753f127fSDimitry Andric DebuginfodCollection::DebuginfodCollection(ArrayRef<StringRef> PathsRef, 351*0fca6ea1SDimitry Andric DebuginfodLog &Log, 352*0fca6ea1SDimitry Andric ThreadPoolInterface &Pool, 353753f127fSDimitry Andric double MinInterval) 354753f127fSDimitry Andric : Log(Log), Pool(Pool), MinInterval(MinInterval) { 355753f127fSDimitry Andric for (StringRef Path : PathsRef) 356753f127fSDimitry Andric Paths.push_back(Path.str()); 357753f127fSDimitry Andric } 358753f127fSDimitry Andric 359753f127fSDimitry Andric Error DebuginfodCollection::update() { 360753f127fSDimitry Andric std::lock_guard<sys::Mutex> Guard(UpdateMutex); 361753f127fSDimitry Andric if (UpdateTimer.isRunning()) 362753f127fSDimitry Andric UpdateTimer.stopTimer(); 363753f127fSDimitry Andric UpdateTimer.clear(); 364753f127fSDimitry Andric for (const std::string &Path : Paths) { 365753f127fSDimitry Andric Log.push("Updating binaries at path " + Path); 366753f127fSDimitry Andric if (Error Err = findBinaries(Path)) 367753f127fSDimitry Andric return Err; 368753f127fSDimitry Andric } 369753f127fSDimitry Andric Log.push("Updated collection"); 370753f127fSDimitry Andric UpdateTimer.startTimer(); 371753f127fSDimitry Andric return Error::success(); 372753f127fSDimitry Andric } 373753f127fSDimitry Andric 374753f127fSDimitry Andric Expected<bool> DebuginfodCollection::updateIfStale() { 375753f127fSDimitry Andric if (!UpdateTimer.isRunning()) 376753f127fSDimitry Andric return false; 377753f127fSDimitry Andric UpdateTimer.stopTimer(); 378753f127fSDimitry Andric double Time = UpdateTimer.getTotalTime().getWallTime(); 379753f127fSDimitry Andric UpdateTimer.startTimer(); 380753f127fSDimitry Andric if (Time < MinInterval) 381753f127fSDimitry Andric return false; 382753f127fSDimitry Andric if (Error Err = update()) 383753f127fSDimitry Andric return std::move(Err); 384753f127fSDimitry Andric return true; 385753f127fSDimitry Andric } 386753f127fSDimitry Andric 387753f127fSDimitry Andric Error DebuginfodCollection::updateForever(std::chrono::milliseconds Interval) { 388753f127fSDimitry Andric while (true) { 389753f127fSDimitry Andric if (Error Err = update()) 390753f127fSDimitry Andric return Err; 391753f127fSDimitry Andric std::this_thread::sleep_for(Interval); 392753f127fSDimitry Andric } 393753f127fSDimitry Andric llvm_unreachable("updateForever loop should never end"); 394753f127fSDimitry Andric } 395753f127fSDimitry Andric 396753f127fSDimitry Andric static bool hasELFMagic(StringRef FilePath) { 397753f127fSDimitry Andric file_magic Type; 398753f127fSDimitry Andric std::error_code EC = identify_magic(FilePath, Type); 399753f127fSDimitry Andric if (EC) 400753f127fSDimitry Andric return false; 401753f127fSDimitry Andric switch (Type) { 402753f127fSDimitry Andric case file_magic::elf: 403753f127fSDimitry Andric case file_magic::elf_relocatable: 404753f127fSDimitry Andric case file_magic::elf_executable: 405753f127fSDimitry Andric case file_magic::elf_shared_object: 406753f127fSDimitry Andric case file_magic::elf_core: 407753f127fSDimitry Andric return true; 408753f127fSDimitry Andric default: 409753f127fSDimitry Andric return false; 410753f127fSDimitry Andric } 411753f127fSDimitry Andric } 412753f127fSDimitry Andric 413753f127fSDimitry Andric Error DebuginfodCollection::findBinaries(StringRef Path) { 414753f127fSDimitry Andric std::error_code EC; 415753f127fSDimitry Andric sys::fs::recursive_directory_iterator I(Twine(Path), EC), E; 416753f127fSDimitry Andric std::mutex IteratorMutex; 417753f127fSDimitry Andric ThreadPoolTaskGroup IteratorGroup(Pool); 418*0fca6ea1SDimitry Andric for (unsigned WorkerIndex = 0; WorkerIndex < Pool.getMaxConcurrency(); 419753f127fSDimitry Andric WorkerIndex++) { 420753f127fSDimitry Andric IteratorGroup.async([&, this]() -> void { 421753f127fSDimitry Andric std::string FilePath; 422753f127fSDimitry Andric while (true) { 423753f127fSDimitry Andric { 424753f127fSDimitry Andric // Check if iteration is over or there is an error during iteration 425753f127fSDimitry Andric std::lock_guard<std::mutex> Guard(IteratorMutex); 426753f127fSDimitry Andric if (I == E || EC) 427753f127fSDimitry Andric return; 428753f127fSDimitry Andric // Grab a file path from the directory iterator and advance the 429753f127fSDimitry Andric // iterator. 430753f127fSDimitry Andric FilePath = I->path(); 431753f127fSDimitry Andric I.increment(EC); 432753f127fSDimitry Andric } 433753f127fSDimitry Andric 434753f127fSDimitry Andric // Inspect the file at this path to determine if it is debuginfo. 435753f127fSDimitry Andric if (!hasELFMagic(FilePath)) 436753f127fSDimitry Andric continue; 437753f127fSDimitry Andric 438753f127fSDimitry Andric Expected<object::OwningBinary<object::Binary>> BinOrErr = 439753f127fSDimitry Andric object::createBinary(FilePath); 440753f127fSDimitry Andric 441753f127fSDimitry Andric if (!BinOrErr) { 442753f127fSDimitry Andric consumeError(BinOrErr.takeError()); 443753f127fSDimitry Andric continue; 444753f127fSDimitry Andric } 445753f127fSDimitry Andric object::Binary *Bin = std::move(BinOrErr.get().getBinary()); 446753f127fSDimitry Andric if (!Bin->isObject()) 447753f127fSDimitry Andric continue; 448753f127fSDimitry Andric 449753f127fSDimitry Andric // TODO: Support non-ELF binaries 450753f127fSDimitry Andric object::ELFObjectFileBase *Object = 451753f127fSDimitry Andric dyn_cast<object::ELFObjectFileBase>(Bin); 452753f127fSDimitry Andric if (!Object) 453753f127fSDimitry Andric continue; 454753f127fSDimitry Andric 45506c3fb27SDimitry Andric BuildIDRef ID = getBuildID(Object); 45606c3fb27SDimitry Andric if (ID.empty()) 457753f127fSDimitry Andric continue; 458753f127fSDimitry Andric 45906c3fb27SDimitry Andric std::string IDString = buildIDToString(ID); 460bdd1243dSDimitry Andric if (Object->hasDebugInfo()) { 461753f127fSDimitry Andric std::lock_guard<sys::RWMutex> DebugBinariesGuard(DebugBinariesMutex); 462bdd1243dSDimitry Andric (void)DebugBinaries.try_emplace(IDString, std::move(FilePath)); 463753f127fSDimitry Andric } else { 464753f127fSDimitry Andric std::lock_guard<sys::RWMutex> BinariesGuard(BinariesMutex); 465bdd1243dSDimitry Andric (void)Binaries.try_emplace(IDString, std::move(FilePath)); 466753f127fSDimitry Andric } 467753f127fSDimitry Andric } 468753f127fSDimitry Andric }); 469753f127fSDimitry Andric } 470753f127fSDimitry Andric IteratorGroup.wait(); 471753f127fSDimitry Andric std::unique_lock<std::mutex> Guard(IteratorMutex); 472753f127fSDimitry Andric if (EC) 473753f127fSDimitry Andric return errorCodeToError(EC); 474753f127fSDimitry Andric return Error::success(); 475753f127fSDimitry Andric } 476753f127fSDimitry Andric 477bdd1243dSDimitry Andric Expected<std::optional<std::string>> 478753f127fSDimitry Andric DebuginfodCollection::getBinaryPath(BuildIDRef ID) { 479753f127fSDimitry Andric Log.push("getting binary path of ID " + buildIDToString(ID)); 480753f127fSDimitry Andric std::shared_lock<sys::RWMutex> Guard(BinariesMutex); 481753f127fSDimitry Andric auto Loc = Binaries.find(buildIDToString(ID)); 482753f127fSDimitry Andric if (Loc != Binaries.end()) { 483753f127fSDimitry Andric std::string Path = Loc->getValue(); 484753f127fSDimitry Andric return Path; 485753f127fSDimitry Andric } 486bdd1243dSDimitry Andric return std::nullopt; 487753f127fSDimitry Andric } 488753f127fSDimitry Andric 489bdd1243dSDimitry Andric Expected<std::optional<std::string>> 490753f127fSDimitry Andric DebuginfodCollection::getDebugBinaryPath(BuildIDRef ID) { 491753f127fSDimitry Andric Log.push("getting debug binary path of ID " + buildIDToString(ID)); 492753f127fSDimitry Andric std::shared_lock<sys::RWMutex> Guard(DebugBinariesMutex); 493753f127fSDimitry Andric auto Loc = DebugBinaries.find(buildIDToString(ID)); 494753f127fSDimitry Andric if (Loc != DebugBinaries.end()) { 495753f127fSDimitry Andric std::string Path = Loc->getValue(); 496753f127fSDimitry Andric return Path; 497753f127fSDimitry Andric } 498bdd1243dSDimitry Andric return std::nullopt; 499753f127fSDimitry Andric } 500753f127fSDimitry Andric 501753f127fSDimitry Andric Expected<std::string> DebuginfodCollection::findBinaryPath(BuildIDRef ID) { 502753f127fSDimitry Andric { 503753f127fSDimitry Andric // Check collection; perform on-demand update if stale. 504bdd1243dSDimitry Andric Expected<std::optional<std::string>> PathOrErr = getBinaryPath(ID); 505753f127fSDimitry Andric if (!PathOrErr) 506753f127fSDimitry Andric return PathOrErr.takeError(); 507bdd1243dSDimitry Andric std::optional<std::string> Path = *PathOrErr; 508753f127fSDimitry Andric if (!Path) { 509753f127fSDimitry Andric Expected<bool> UpdatedOrErr = updateIfStale(); 510753f127fSDimitry Andric if (!UpdatedOrErr) 511753f127fSDimitry Andric return UpdatedOrErr.takeError(); 512753f127fSDimitry Andric if (*UpdatedOrErr) { 513753f127fSDimitry Andric // Try once more. 514753f127fSDimitry Andric PathOrErr = getBinaryPath(ID); 515753f127fSDimitry Andric if (!PathOrErr) 516753f127fSDimitry Andric return PathOrErr.takeError(); 517753f127fSDimitry Andric Path = *PathOrErr; 518753f127fSDimitry Andric } 519753f127fSDimitry Andric } 520753f127fSDimitry Andric if (Path) 521bdd1243dSDimitry Andric return *Path; 522753f127fSDimitry Andric } 523753f127fSDimitry Andric 524753f127fSDimitry Andric // Try federation. 525753f127fSDimitry Andric Expected<std::string> PathOrErr = getCachedOrDownloadExecutable(ID); 526753f127fSDimitry Andric if (!PathOrErr) 527753f127fSDimitry Andric consumeError(PathOrErr.takeError()); 528753f127fSDimitry Andric 529753f127fSDimitry Andric // Fall back to debug binary. 530753f127fSDimitry Andric return findDebugBinaryPath(ID); 531753f127fSDimitry Andric } 532753f127fSDimitry Andric 533753f127fSDimitry Andric Expected<std::string> DebuginfodCollection::findDebugBinaryPath(BuildIDRef ID) { 534753f127fSDimitry Andric // Check collection; perform on-demand update if stale. 535bdd1243dSDimitry Andric Expected<std::optional<std::string>> PathOrErr = getDebugBinaryPath(ID); 536753f127fSDimitry Andric if (!PathOrErr) 537753f127fSDimitry Andric return PathOrErr.takeError(); 538bdd1243dSDimitry Andric std::optional<std::string> Path = *PathOrErr; 539753f127fSDimitry Andric if (!Path) { 540753f127fSDimitry Andric Expected<bool> UpdatedOrErr = updateIfStale(); 541753f127fSDimitry Andric if (!UpdatedOrErr) 542753f127fSDimitry Andric return UpdatedOrErr.takeError(); 543753f127fSDimitry Andric if (*UpdatedOrErr) { 544753f127fSDimitry Andric // Try once more. 545753f127fSDimitry Andric PathOrErr = getBinaryPath(ID); 546753f127fSDimitry Andric if (!PathOrErr) 547753f127fSDimitry Andric return PathOrErr.takeError(); 548753f127fSDimitry Andric Path = *PathOrErr; 549753f127fSDimitry Andric } 550753f127fSDimitry Andric } 551753f127fSDimitry Andric if (Path) 552bdd1243dSDimitry Andric return *Path; 553753f127fSDimitry Andric 554753f127fSDimitry Andric // Try federation. 555753f127fSDimitry Andric return getCachedOrDownloadDebuginfo(ID); 556753f127fSDimitry Andric } 557753f127fSDimitry Andric 558753f127fSDimitry Andric DebuginfodServer::DebuginfodServer(DebuginfodLog &Log, 559753f127fSDimitry Andric DebuginfodCollection &Collection) 560753f127fSDimitry Andric : Log(Log), Collection(Collection) { 561753f127fSDimitry Andric cantFail( 562753f127fSDimitry Andric Server.get(R"(/buildid/(.*)/debuginfo)", [&](HTTPServerRequest Request) { 563753f127fSDimitry Andric Log.push("GET " + Request.UrlPath); 564753f127fSDimitry Andric std::string IDString; 565753f127fSDimitry Andric if (!tryGetFromHex(Request.UrlPathMatches[0], IDString)) { 566753f127fSDimitry Andric Request.setResponse( 567753f127fSDimitry Andric {404, "text/plain", "Build ID is not a hex string\n"}); 568753f127fSDimitry Andric return; 569753f127fSDimitry Andric } 570bdd1243dSDimitry Andric object::BuildID ID(IDString.begin(), IDString.end()); 571753f127fSDimitry Andric Expected<std::string> PathOrErr = Collection.findDebugBinaryPath(ID); 572753f127fSDimitry Andric if (Error Err = PathOrErr.takeError()) { 573753f127fSDimitry Andric consumeError(std::move(Err)); 574753f127fSDimitry Andric Request.setResponse({404, "text/plain", "Build ID not found\n"}); 575753f127fSDimitry Andric return; 576753f127fSDimitry Andric } 577753f127fSDimitry Andric streamFile(Request, *PathOrErr); 578753f127fSDimitry Andric })); 579753f127fSDimitry Andric cantFail( 580753f127fSDimitry Andric Server.get(R"(/buildid/(.*)/executable)", [&](HTTPServerRequest Request) { 581753f127fSDimitry Andric Log.push("GET " + Request.UrlPath); 582753f127fSDimitry Andric std::string IDString; 583753f127fSDimitry Andric if (!tryGetFromHex(Request.UrlPathMatches[0], IDString)) { 584753f127fSDimitry Andric Request.setResponse( 585753f127fSDimitry Andric {404, "text/plain", "Build ID is not a hex string\n"}); 586753f127fSDimitry Andric return; 587753f127fSDimitry Andric } 588bdd1243dSDimitry Andric object::BuildID ID(IDString.begin(), IDString.end()); 589753f127fSDimitry Andric Expected<std::string> PathOrErr = Collection.findBinaryPath(ID); 590753f127fSDimitry Andric if (Error Err = PathOrErr.takeError()) { 591753f127fSDimitry Andric consumeError(std::move(Err)); 592753f127fSDimitry Andric Request.setResponse({404, "text/plain", "Build ID not found\n"}); 593753f127fSDimitry Andric return; 594753f127fSDimitry Andric } 595753f127fSDimitry Andric streamFile(Request, *PathOrErr); 596753f127fSDimitry Andric })); 597753f127fSDimitry Andric } 598753f127fSDimitry Andric 5990eae32dcSDimitry Andric } // namespace llvm 600