xref: /freebsd-src/contrib/llvm-project/llvm/lib/Debuginfod/Debuginfod.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
10eae32dcSDimitry Andric //===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===//
20eae32dcSDimitry Andric //
30eae32dcSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40eae32dcSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50eae32dcSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60eae32dcSDimitry Andric //
70eae32dcSDimitry Andric //===----------------------------------------------------------------------===//
80eae32dcSDimitry Andric ///
90eae32dcSDimitry Andric /// \file
100eae32dcSDimitry Andric ///
11753f127fSDimitry Andric /// This file contains several definitions for the debuginfod client and server.
12753f127fSDimitry Andric /// For the client, this file defines the fetchInfo function. For the server,
13753f127fSDimitry Andric /// this file defines the DebuginfodLogEntry and DebuginfodServer structs, as
14753f127fSDimitry Andric /// well as the DebuginfodLog, DebuginfodCollection classes. The fetchInfo
15753f127fSDimitry Andric /// function retrieves any of the three supported artifact types: (executable,
16753f127fSDimitry Andric /// debuginfo, source file) associated with a build-id from debuginfod servers.
17753f127fSDimitry Andric /// If a source file is to be fetched, its absolute path must be specified in
18753f127fSDimitry Andric /// the Description argument to fetchInfo. The DebuginfodLogEntry,
19753f127fSDimitry Andric /// DebuginfodLog, and DebuginfodCollection are used by the DebuginfodServer to
20753f127fSDimitry Andric /// scan the local filesystem for binaries and serve the debuginfod protocol.
210eae32dcSDimitry Andric ///
220eae32dcSDimitry Andric //===----------------------------------------------------------------------===//
230eae32dcSDimitry Andric 
240eae32dcSDimitry Andric #include "llvm/Debuginfod/Debuginfod.h"
25bdd1243dSDimitry Andric #include "llvm/ADT/StringExtras.h"
260eae32dcSDimitry Andric #include "llvm/ADT/StringRef.h"
27753f127fSDimitry Andric #include "llvm/BinaryFormat/Magic.h"
28753f127fSDimitry Andric #include "llvm/DebugInfo/DWARF/DWARFContext.h"
29753f127fSDimitry Andric #include "llvm/DebugInfo/Symbolize/Symbolize.h"
300eae32dcSDimitry Andric #include "llvm/Debuginfod/HTTPClient.h"
31bdd1243dSDimitry Andric #include "llvm/Object/BuildID.h"
32753f127fSDimitry Andric #include "llvm/Object/ELFObjectFile.h"
330eae32dcSDimitry Andric #include "llvm/Support/CachePruning.h"
340eae32dcSDimitry Andric #include "llvm/Support/Caching.h"
3504eeddc0SDimitry Andric #include "llvm/Support/Errc.h"
360eae32dcSDimitry Andric #include "llvm/Support/Error.h"
370eae32dcSDimitry Andric #include "llvm/Support/FileUtilities.h"
38bdd1243dSDimitry Andric #include "llvm/Support/MemoryBuffer.h"
3904eeddc0SDimitry Andric #include "llvm/Support/Path.h"
40753f127fSDimitry Andric #include "llvm/Support/ThreadPool.h"
410eae32dcSDimitry Andric #include "llvm/Support/xxhash.h"
420eae32dcSDimitry Andric 
43753f127fSDimitry Andric #include <atomic>
445f757f3fSDimitry Andric #include <optional>
45bdd1243dSDimitry Andric #include <thread>
46753f127fSDimitry Andric 
470eae32dcSDimitry Andric namespace llvm {
48bdd1243dSDimitry Andric 
49bdd1243dSDimitry Andric using llvm::object::BuildIDRef;
50bdd1243dSDimitry Andric 
515f757f3fSDimitry Andric namespace {
525f757f3fSDimitry Andric std::optional<SmallVector<StringRef>> DebuginfodUrls;
535f757f3fSDimitry Andric // Many Readers/Single Writer lock protecting the global debuginfod URL list.
545f757f3fSDimitry Andric llvm::sys::RWMutex UrlsMutex;
555f757f3fSDimitry Andric } // namespace
565f757f3fSDimitry Andric 
577a6dacacSDimitry Andric std::string getDebuginfodCacheKey(llvm::StringRef S) {
585f757f3fSDimitry Andric   return utostr(xxh3_64bits(S));
595f757f3fSDimitry Andric }
600eae32dcSDimitry Andric 
610eae32dcSDimitry Andric // Returns a binary BuildID as a normalized hex string.
620eae32dcSDimitry Andric // Uses lowercase for compatibility with common debuginfod servers.
630eae32dcSDimitry Andric static std::string buildIDToString(BuildIDRef ID) {
640eae32dcSDimitry Andric   return llvm::toHex(ID, /*LowerCase=*/true);
650eae32dcSDimitry Andric }
660eae32dcSDimitry Andric 
671ac55f4cSDimitry Andric bool canUseDebuginfod() {
681ac55f4cSDimitry Andric   return HTTPClient::isAvailable() && !getDefaultDebuginfodUrls().empty();
691ac55f4cSDimitry Andric }
701ac55f4cSDimitry Andric 
711ac55f4cSDimitry Andric SmallVector<StringRef> getDefaultDebuginfodUrls() {
725f757f3fSDimitry Andric   std::shared_lock<llvm::sys::RWMutex> ReadGuard(UrlsMutex);
735f757f3fSDimitry Andric   if (!DebuginfodUrls) {
745f757f3fSDimitry Andric     // Only read from the environment variable if the user hasn't already
757a6dacacSDimitry Andric     // set the value.
765f757f3fSDimitry Andric     ReadGuard.unlock();
775f757f3fSDimitry Andric     std::unique_lock<llvm::sys::RWMutex> WriteGuard(UrlsMutex);
785f757f3fSDimitry Andric     DebuginfodUrls = SmallVector<StringRef>();
795f757f3fSDimitry Andric     if (const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS")) {
805f757f3fSDimitry Andric       StringRef(DebuginfodUrlsEnv)
815f757f3fSDimitry Andric           .split(DebuginfodUrls.value(), " ", -1, false);
825f757f3fSDimitry Andric     }
835f757f3fSDimitry Andric     WriteGuard.unlock();
845f757f3fSDimitry Andric     ReadGuard.lock();
855f757f3fSDimitry Andric   }
865f757f3fSDimitry Andric   return DebuginfodUrls.value();
875f757f3fSDimitry Andric }
880eae32dcSDimitry Andric 
897a6dacacSDimitry Andric // Set the default debuginfod URL list, override the environment variable.
905f757f3fSDimitry Andric void setDefaultDebuginfodUrls(const SmallVector<StringRef> &URLs) {
915f757f3fSDimitry Andric   std::unique_lock<llvm::sys::RWMutex> WriteGuard(UrlsMutex);
925f757f3fSDimitry Andric   DebuginfodUrls = URLs;
930eae32dcSDimitry Andric }
940eae32dcSDimitry Andric 
95753f127fSDimitry Andric /// Finds a default local file caching directory for the debuginfod client,
96753f127fSDimitry Andric /// first checking DEBUGINFOD_CACHE_PATH.
970eae32dcSDimitry Andric Expected<std::string> getDefaultDebuginfodCacheDirectory() {
980eae32dcSDimitry Andric   if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH"))
990eae32dcSDimitry Andric     return CacheDirectoryEnv;
1000eae32dcSDimitry Andric 
1010eae32dcSDimitry Andric   SmallString<64> CacheDirectory;
1020eae32dcSDimitry Andric   if (!sys::path::cache_directory(CacheDirectory))
1030eae32dcSDimitry Andric     return createStringError(
1040eae32dcSDimitry Andric         errc::io_error, "Unable to determine appropriate cache directory.");
10504eeddc0SDimitry Andric   sys::path::append(CacheDirectory, "llvm-debuginfod", "client");
1060eae32dcSDimitry Andric   return std::string(CacheDirectory);
1070eae32dcSDimitry Andric }
1080eae32dcSDimitry Andric 
1090eae32dcSDimitry Andric std::chrono::milliseconds getDefaultDebuginfodTimeout() {
1100eae32dcSDimitry Andric   long Timeout;
1110eae32dcSDimitry Andric   const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT");
1120eae32dcSDimitry Andric   if (DebuginfodTimeoutEnv &&
1130eae32dcSDimitry Andric       to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10))
1140eae32dcSDimitry Andric     return std::chrono::milliseconds(Timeout * 1000);
1150eae32dcSDimitry Andric 
1160eae32dcSDimitry Andric   return std::chrono::milliseconds(90 * 1000);
1170eae32dcSDimitry Andric }
1180eae32dcSDimitry Andric 
1190eae32dcSDimitry Andric /// The following functions fetch a debuginfod artifact to a file in a local
1200eae32dcSDimitry Andric /// cache and return the cached file path. They first search the local cache,
1210eae32dcSDimitry Andric /// followed by the debuginfod servers.
1220eae32dcSDimitry Andric 
1237a6dacacSDimitry Andric std::string getDebuginfodSourceUrlPath(BuildIDRef ID,
1240eae32dcSDimitry Andric                                        StringRef SourceFilePath) {
1250eae32dcSDimitry Andric   SmallString<64> UrlPath;
1260eae32dcSDimitry Andric   sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
1270eae32dcSDimitry Andric                     buildIDToString(ID), "source",
1280eae32dcSDimitry Andric                     sys::path::convert_to_slash(SourceFilePath));
1297a6dacacSDimitry Andric   return std::string(UrlPath);
1300eae32dcSDimitry Andric }
1310eae32dcSDimitry Andric 
1327a6dacacSDimitry Andric Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
1337a6dacacSDimitry Andric                                                 StringRef SourceFilePath) {
1347a6dacacSDimitry Andric   std::string UrlPath = getDebuginfodSourceUrlPath(ID, SourceFilePath);
1357a6dacacSDimitry Andric   return getCachedOrDownloadArtifact(getDebuginfodCacheKey(UrlPath), UrlPath);
1367a6dacacSDimitry Andric }
1377a6dacacSDimitry Andric 
1387a6dacacSDimitry Andric std::string getDebuginfodExecutableUrlPath(BuildIDRef ID) {
1390eae32dcSDimitry Andric   SmallString<64> UrlPath;
1400eae32dcSDimitry Andric   sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
1410eae32dcSDimitry Andric                     buildIDToString(ID), "executable");
1427a6dacacSDimitry Andric   return std::string(UrlPath);
1430eae32dcSDimitry Andric }
1440eae32dcSDimitry Andric 
1457a6dacacSDimitry Andric Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) {
1467a6dacacSDimitry Andric   std::string UrlPath = getDebuginfodExecutableUrlPath(ID);
1477a6dacacSDimitry Andric   return getCachedOrDownloadArtifact(getDebuginfodCacheKey(UrlPath), UrlPath);
1487a6dacacSDimitry Andric }
1497a6dacacSDimitry Andric 
1507a6dacacSDimitry Andric std::string getDebuginfodDebuginfoUrlPath(BuildIDRef ID) {
1510eae32dcSDimitry Andric   SmallString<64> UrlPath;
1520eae32dcSDimitry Andric   sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
1530eae32dcSDimitry Andric                     buildIDToString(ID), "debuginfo");
1547a6dacacSDimitry Andric   return std::string(UrlPath);
1557a6dacacSDimitry Andric }
1567a6dacacSDimitry Andric 
1577a6dacacSDimitry Andric Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) {
1587a6dacacSDimitry Andric   std::string UrlPath = getDebuginfodDebuginfoUrlPath(ID);
1597a6dacacSDimitry Andric   return getCachedOrDownloadArtifact(getDebuginfodCacheKey(UrlPath), UrlPath);
1600eae32dcSDimitry Andric }
1610eae32dcSDimitry Andric 
1620eae32dcSDimitry Andric // General fetching function.
1630eae32dcSDimitry Andric Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
1640eae32dcSDimitry Andric                                                   StringRef UrlPath) {
1650eae32dcSDimitry Andric   SmallString<10> CacheDir;
1660eae32dcSDimitry Andric 
1670eae32dcSDimitry Andric   Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory();
1680eae32dcSDimitry Andric   if (!CacheDirOrErr)
1690eae32dcSDimitry Andric     return CacheDirOrErr.takeError();
1700eae32dcSDimitry Andric   CacheDir = *CacheDirOrErr;
1710eae32dcSDimitry Andric 
1720eae32dcSDimitry Andric   return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
1731ac55f4cSDimitry Andric                                      getDefaultDebuginfodUrls(),
1740eae32dcSDimitry Andric                                      getDefaultDebuginfodTimeout());
1750eae32dcSDimitry Andric }
1760eae32dcSDimitry Andric 
17781ad6265SDimitry Andric namespace {
17881ad6265SDimitry Andric 
17981ad6265SDimitry Andric /// A simple handler which streams the returned data to a cache file. The cache
18081ad6265SDimitry Andric /// file is only created if a 200 OK status is observed.
18181ad6265SDimitry Andric class StreamedHTTPResponseHandler : public HTTPResponseHandler {
18281ad6265SDimitry Andric   using CreateStreamFn =
18381ad6265SDimitry Andric       std::function<Expected<std::unique_ptr<CachedFileStream>>()>;
18481ad6265SDimitry Andric   CreateStreamFn CreateStream;
18581ad6265SDimitry Andric   HTTPClient &Client;
18681ad6265SDimitry Andric   std::unique_ptr<CachedFileStream> FileStream;
18781ad6265SDimitry Andric 
18881ad6265SDimitry Andric public:
18981ad6265SDimitry Andric   StreamedHTTPResponseHandler(CreateStreamFn CreateStream, HTTPClient &Client)
19081ad6265SDimitry Andric       : CreateStream(CreateStream), Client(Client) {}
19181ad6265SDimitry Andric   virtual ~StreamedHTTPResponseHandler() = default;
19281ad6265SDimitry Andric 
19381ad6265SDimitry Andric   Error handleBodyChunk(StringRef BodyChunk) override;
19481ad6265SDimitry Andric };
19581ad6265SDimitry Andric 
19681ad6265SDimitry Andric } // namespace
19781ad6265SDimitry Andric 
19881ad6265SDimitry Andric Error StreamedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) {
19981ad6265SDimitry Andric   if (!FileStream) {
2001ac55f4cSDimitry Andric     unsigned Code = Client.responseCode();
2011ac55f4cSDimitry Andric     if (Code && Code != 200)
20281ad6265SDimitry Andric       return Error::success();
20381ad6265SDimitry Andric     Expected<std::unique_ptr<CachedFileStream>> FileStreamOrError =
20481ad6265SDimitry Andric         CreateStream();
20581ad6265SDimitry Andric     if (!FileStreamOrError)
20681ad6265SDimitry Andric       return FileStreamOrError.takeError();
20781ad6265SDimitry Andric     FileStream = std::move(*FileStreamOrError);
20881ad6265SDimitry Andric   }
20981ad6265SDimitry Andric   *FileStream->OS << BodyChunk;
21081ad6265SDimitry Andric   return Error::success();
21181ad6265SDimitry Andric }
21281ad6265SDimitry Andric 
213bdd1243dSDimitry Andric // An over-accepting simplification of the HTTP RFC 7230 spec.
214bdd1243dSDimitry Andric static bool isHeader(StringRef S) {
215bdd1243dSDimitry Andric   StringRef Name;
216bdd1243dSDimitry Andric   StringRef Value;
217bdd1243dSDimitry Andric   std::tie(Name, Value) = S.split(':');
218bdd1243dSDimitry Andric   if (Name.empty() || Value.empty())
219bdd1243dSDimitry Andric     return false;
220bdd1243dSDimitry Andric   return all_of(Name, [](char C) { return llvm::isPrint(C) && C != ' '; }) &&
221bdd1243dSDimitry Andric          all_of(Value, [](char C) { return llvm::isPrint(C) || C == '\t'; });
222bdd1243dSDimitry Andric }
223bdd1243dSDimitry Andric 
224bdd1243dSDimitry Andric static SmallVector<std::string, 0> getHeaders() {
225bdd1243dSDimitry Andric   const char *Filename = getenv("DEBUGINFOD_HEADERS_FILE");
226bdd1243dSDimitry Andric   if (!Filename)
227bdd1243dSDimitry Andric     return {};
228bdd1243dSDimitry Andric   ErrorOr<std::unique_ptr<MemoryBuffer>> HeadersFile =
229bdd1243dSDimitry Andric       MemoryBuffer::getFile(Filename, /*IsText=*/true);
230bdd1243dSDimitry Andric   if (!HeadersFile)
231bdd1243dSDimitry Andric     return {};
232bdd1243dSDimitry Andric 
233bdd1243dSDimitry Andric   SmallVector<std::string, 0> Headers;
234bdd1243dSDimitry Andric   uint64_t LineNumber = 0;
235bdd1243dSDimitry Andric   for (StringRef Line : llvm::split((*HeadersFile)->getBuffer(), '\n')) {
236bdd1243dSDimitry Andric     LineNumber++;
237bdd1243dSDimitry Andric     if (!Line.empty() && Line.back() == '\r')
238bdd1243dSDimitry Andric       Line = Line.drop_back();
239bdd1243dSDimitry Andric     if (!isHeader(Line)) {
240bdd1243dSDimitry Andric       if (!all_of(Line, llvm::isSpace))
241bdd1243dSDimitry Andric         WithColor::warning()
242bdd1243dSDimitry Andric             << "could not parse debuginfod header: " << Filename << ':'
243bdd1243dSDimitry Andric             << LineNumber << '\n';
244bdd1243dSDimitry Andric       continue;
245bdd1243dSDimitry Andric     }
246bdd1243dSDimitry Andric     Headers.emplace_back(Line);
247bdd1243dSDimitry Andric   }
248bdd1243dSDimitry Andric   return Headers;
249bdd1243dSDimitry Andric }
250bdd1243dSDimitry Andric 
2510eae32dcSDimitry Andric Expected<std::string> getCachedOrDownloadArtifact(
2520eae32dcSDimitry Andric     StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
2530eae32dcSDimitry Andric     ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) {
2540eae32dcSDimitry Andric   SmallString<64> AbsCachedArtifactPath;
2550eae32dcSDimitry Andric   sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath,
2560eae32dcSDimitry Andric                     "llvmcache-" + UniqueKey);
2570eae32dcSDimitry Andric 
2580eae32dcSDimitry Andric   Expected<FileCache> CacheOrErr =
2590eae32dcSDimitry Andric       localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath);
2600eae32dcSDimitry Andric   if (!CacheOrErr)
2610eae32dcSDimitry Andric     return CacheOrErr.takeError();
2620eae32dcSDimitry Andric 
2630eae32dcSDimitry Andric   FileCache Cache = *CacheOrErr;
2640eae32dcSDimitry Andric   // We choose an arbitrary Task parameter as we do not make use of it.
2650eae32dcSDimitry Andric   unsigned Task = 0;
266bdd1243dSDimitry Andric   Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey, "");
2670eae32dcSDimitry Andric   if (!CacheAddStreamOrErr)
2680eae32dcSDimitry Andric     return CacheAddStreamOrErr.takeError();
2690eae32dcSDimitry Andric   AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
2700eae32dcSDimitry Andric   if (!CacheAddStream)
2710eae32dcSDimitry Andric     return std::string(AbsCachedArtifactPath);
2720eae32dcSDimitry Andric   // The artifact was not found in the local cache, query the debuginfod
2730eae32dcSDimitry Andric   // servers.
2740eae32dcSDimitry Andric   if (!HTTPClient::isAvailable())
2750eae32dcSDimitry Andric     return createStringError(errc::io_error,
2760eae32dcSDimitry Andric                              "No working HTTP client is available.");
2770eae32dcSDimitry Andric 
2780eae32dcSDimitry Andric   if (!HTTPClient::IsInitialized)
2790eae32dcSDimitry Andric     return createStringError(
2800eae32dcSDimitry Andric         errc::io_error,
2810eae32dcSDimitry Andric         "A working HTTP client is available, but it is not initialized. To "
2820eae32dcSDimitry Andric         "allow Debuginfod to make HTTP requests, call HTTPClient::initialize() "
2830eae32dcSDimitry Andric         "at the beginning of main.");
2840eae32dcSDimitry Andric 
2850eae32dcSDimitry Andric   HTTPClient Client;
2860eae32dcSDimitry Andric   Client.setTimeout(Timeout);
2870eae32dcSDimitry Andric   for (StringRef ServerUrl : DebuginfodUrls) {
2880eae32dcSDimitry Andric     SmallString<64> ArtifactUrl;
2890eae32dcSDimitry Andric     sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath);
2900eae32dcSDimitry Andric 
29181ad6265SDimitry Andric     // Perform the HTTP request and if successful, write the response body to
29281ad6265SDimitry Andric     // the cache.
29306c3fb27SDimitry Andric     {
294bdd1243dSDimitry Andric       StreamedHTTPResponseHandler Handler(
295bdd1243dSDimitry Andric           [&]() { return CacheAddStream(Task, ""); }, Client);
29681ad6265SDimitry Andric       HTTPRequest Request(ArtifactUrl);
297bdd1243dSDimitry Andric       Request.Headers = getHeaders();
29881ad6265SDimitry Andric       Error Err = Client.perform(Request, Handler);
29981ad6265SDimitry Andric       if (Err)
30081ad6265SDimitry Andric         return std::move(Err);
3010eae32dcSDimitry Andric 
3021ac55f4cSDimitry Andric       unsigned Code = Client.responseCode();
3031ac55f4cSDimitry Andric       if (Code && Code != 200)
3040eae32dcSDimitry Andric         continue;
30506c3fb27SDimitry Andric     }
30606c3fb27SDimitry Andric 
30706c3fb27SDimitry Andric     Expected<CachePruningPolicy> PruningPolicyOrErr =
30806c3fb27SDimitry Andric         parseCachePruningPolicy(std::getenv("DEBUGINFOD_CACHE_POLICY"));
30906c3fb27SDimitry Andric     if (!PruningPolicyOrErr)
31006c3fb27SDimitry Andric       return PruningPolicyOrErr.takeError();
31106c3fb27SDimitry Andric     pruneCache(CacheDirectoryPath, *PruningPolicyOrErr);
3120eae32dcSDimitry Andric 
3130eae32dcSDimitry Andric     // Return the path to the artifact on disk.
3140eae32dcSDimitry Andric     return std::string(AbsCachedArtifactPath);
3150eae32dcSDimitry Andric   }
3160eae32dcSDimitry Andric 
3170eae32dcSDimitry Andric   return createStringError(errc::argument_out_of_domain, "build id not found");
3180eae32dcSDimitry Andric }
319753f127fSDimitry Andric 
320753f127fSDimitry Andric DebuginfodLogEntry::DebuginfodLogEntry(const Twine &Message)
321753f127fSDimitry Andric     : Message(Message.str()) {}
322753f127fSDimitry Andric 
323753f127fSDimitry Andric void DebuginfodLog::push(const Twine &Message) {
324753f127fSDimitry Andric   push(DebuginfodLogEntry(Message));
325753f127fSDimitry Andric }
326753f127fSDimitry Andric 
327753f127fSDimitry Andric void DebuginfodLog::push(DebuginfodLogEntry Entry) {
328753f127fSDimitry Andric   {
329753f127fSDimitry Andric     std::lock_guard<std::mutex> Guard(QueueMutex);
330753f127fSDimitry Andric     LogEntryQueue.push(Entry);
331753f127fSDimitry Andric   }
332753f127fSDimitry Andric   QueueCondition.notify_one();
333753f127fSDimitry Andric }
334753f127fSDimitry Andric 
335753f127fSDimitry Andric DebuginfodLogEntry DebuginfodLog::pop() {
336753f127fSDimitry Andric   {
337753f127fSDimitry Andric     std::unique_lock<std::mutex> Guard(QueueMutex);
338753f127fSDimitry Andric     // Wait for messages to be pushed into the queue.
339753f127fSDimitry Andric     QueueCondition.wait(Guard, [&] { return !LogEntryQueue.empty(); });
340753f127fSDimitry Andric   }
341753f127fSDimitry Andric   std::lock_guard<std::mutex> Guard(QueueMutex);
342753f127fSDimitry Andric   if (!LogEntryQueue.size())
343753f127fSDimitry Andric     llvm_unreachable("Expected message in the queue.");
344753f127fSDimitry Andric 
345753f127fSDimitry Andric   DebuginfodLogEntry Entry = LogEntryQueue.front();
346753f127fSDimitry Andric   LogEntryQueue.pop();
347753f127fSDimitry Andric   return Entry;
348753f127fSDimitry Andric }
349753f127fSDimitry Andric 
350753f127fSDimitry Andric DebuginfodCollection::DebuginfodCollection(ArrayRef<StringRef> PathsRef,
351*0fca6ea1SDimitry Andric                                            DebuginfodLog &Log,
352*0fca6ea1SDimitry Andric                                            ThreadPoolInterface &Pool,
353753f127fSDimitry Andric                                            double MinInterval)
354753f127fSDimitry Andric     : Log(Log), Pool(Pool), MinInterval(MinInterval) {
355753f127fSDimitry Andric   for (StringRef Path : PathsRef)
356753f127fSDimitry Andric     Paths.push_back(Path.str());
357753f127fSDimitry Andric }
358753f127fSDimitry Andric 
359753f127fSDimitry Andric Error DebuginfodCollection::update() {
360753f127fSDimitry Andric   std::lock_guard<sys::Mutex> Guard(UpdateMutex);
361753f127fSDimitry Andric   if (UpdateTimer.isRunning())
362753f127fSDimitry Andric     UpdateTimer.stopTimer();
363753f127fSDimitry Andric   UpdateTimer.clear();
364753f127fSDimitry Andric   for (const std::string &Path : Paths) {
365753f127fSDimitry Andric     Log.push("Updating binaries at path " + Path);
366753f127fSDimitry Andric     if (Error Err = findBinaries(Path))
367753f127fSDimitry Andric       return Err;
368753f127fSDimitry Andric   }
369753f127fSDimitry Andric   Log.push("Updated collection");
370753f127fSDimitry Andric   UpdateTimer.startTimer();
371753f127fSDimitry Andric   return Error::success();
372753f127fSDimitry Andric }
373753f127fSDimitry Andric 
374753f127fSDimitry Andric Expected<bool> DebuginfodCollection::updateIfStale() {
375753f127fSDimitry Andric   if (!UpdateTimer.isRunning())
376753f127fSDimitry Andric     return false;
377753f127fSDimitry Andric   UpdateTimer.stopTimer();
378753f127fSDimitry Andric   double Time = UpdateTimer.getTotalTime().getWallTime();
379753f127fSDimitry Andric   UpdateTimer.startTimer();
380753f127fSDimitry Andric   if (Time < MinInterval)
381753f127fSDimitry Andric     return false;
382753f127fSDimitry Andric   if (Error Err = update())
383753f127fSDimitry Andric     return std::move(Err);
384753f127fSDimitry Andric   return true;
385753f127fSDimitry Andric }
386753f127fSDimitry Andric 
387753f127fSDimitry Andric Error DebuginfodCollection::updateForever(std::chrono::milliseconds Interval) {
388753f127fSDimitry Andric   while (true) {
389753f127fSDimitry Andric     if (Error Err = update())
390753f127fSDimitry Andric       return Err;
391753f127fSDimitry Andric     std::this_thread::sleep_for(Interval);
392753f127fSDimitry Andric   }
393753f127fSDimitry Andric   llvm_unreachable("updateForever loop should never end");
394753f127fSDimitry Andric }
395753f127fSDimitry Andric 
396753f127fSDimitry Andric static bool hasELFMagic(StringRef FilePath) {
397753f127fSDimitry Andric   file_magic Type;
398753f127fSDimitry Andric   std::error_code EC = identify_magic(FilePath, Type);
399753f127fSDimitry Andric   if (EC)
400753f127fSDimitry Andric     return false;
401753f127fSDimitry Andric   switch (Type) {
402753f127fSDimitry Andric   case file_magic::elf:
403753f127fSDimitry Andric   case file_magic::elf_relocatable:
404753f127fSDimitry Andric   case file_magic::elf_executable:
405753f127fSDimitry Andric   case file_magic::elf_shared_object:
406753f127fSDimitry Andric   case file_magic::elf_core:
407753f127fSDimitry Andric     return true;
408753f127fSDimitry Andric   default:
409753f127fSDimitry Andric     return false;
410753f127fSDimitry Andric   }
411753f127fSDimitry Andric }
412753f127fSDimitry Andric 
413753f127fSDimitry Andric Error DebuginfodCollection::findBinaries(StringRef Path) {
414753f127fSDimitry Andric   std::error_code EC;
415753f127fSDimitry Andric   sys::fs::recursive_directory_iterator I(Twine(Path), EC), E;
416753f127fSDimitry Andric   std::mutex IteratorMutex;
417753f127fSDimitry Andric   ThreadPoolTaskGroup IteratorGroup(Pool);
418*0fca6ea1SDimitry Andric   for (unsigned WorkerIndex = 0; WorkerIndex < Pool.getMaxConcurrency();
419753f127fSDimitry Andric        WorkerIndex++) {
420753f127fSDimitry Andric     IteratorGroup.async([&, this]() -> void {
421753f127fSDimitry Andric       std::string FilePath;
422753f127fSDimitry Andric       while (true) {
423753f127fSDimitry Andric         {
424753f127fSDimitry Andric           // Check if iteration is over or there is an error during iteration
425753f127fSDimitry Andric           std::lock_guard<std::mutex> Guard(IteratorMutex);
426753f127fSDimitry Andric           if (I == E || EC)
427753f127fSDimitry Andric             return;
428753f127fSDimitry Andric           // Grab a file path from the directory iterator and advance the
429753f127fSDimitry Andric           // iterator.
430753f127fSDimitry Andric           FilePath = I->path();
431753f127fSDimitry Andric           I.increment(EC);
432753f127fSDimitry Andric         }
433753f127fSDimitry Andric 
434753f127fSDimitry Andric         // Inspect the file at this path to determine if it is debuginfo.
435753f127fSDimitry Andric         if (!hasELFMagic(FilePath))
436753f127fSDimitry Andric           continue;
437753f127fSDimitry Andric 
438753f127fSDimitry Andric         Expected<object::OwningBinary<object::Binary>> BinOrErr =
439753f127fSDimitry Andric             object::createBinary(FilePath);
440753f127fSDimitry Andric 
441753f127fSDimitry Andric         if (!BinOrErr) {
442753f127fSDimitry Andric           consumeError(BinOrErr.takeError());
443753f127fSDimitry Andric           continue;
444753f127fSDimitry Andric         }
445753f127fSDimitry Andric         object::Binary *Bin = std::move(BinOrErr.get().getBinary());
446753f127fSDimitry Andric         if (!Bin->isObject())
447753f127fSDimitry Andric           continue;
448753f127fSDimitry Andric 
449753f127fSDimitry Andric         // TODO: Support non-ELF binaries
450753f127fSDimitry Andric         object::ELFObjectFileBase *Object =
451753f127fSDimitry Andric             dyn_cast<object::ELFObjectFileBase>(Bin);
452753f127fSDimitry Andric         if (!Object)
453753f127fSDimitry Andric           continue;
454753f127fSDimitry Andric 
45506c3fb27SDimitry Andric         BuildIDRef ID = getBuildID(Object);
45606c3fb27SDimitry Andric         if (ID.empty())
457753f127fSDimitry Andric           continue;
458753f127fSDimitry Andric 
45906c3fb27SDimitry Andric         std::string IDString = buildIDToString(ID);
460bdd1243dSDimitry Andric         if (Object->hasDebugInfo()) {
461753f127fSDimitry Andric           std::lock_guard<sys::RWMutex> DebugBinariesGuard(DebugBinariesMutex);
462bdd1243dSDimitry Andric           (void)DebugBinaries.try_emplace(IDString, std::move(FilePath));
463753f127fSDimitry Andric         } else {
464753f127fSDimitry Andric           std::lock_guard<sys::RWMutex> BinariesGuard(BinariesMutex);
465bdd1243dSDimitry Andric           (void)Binaries.try_emplace(IDString, std::move(FilePath));
466753f127fSDimitry Andric         }
467753f127fSDimitry Andric       }
468753f127fSDimitry Andric     });
469753f127fSDimitry Andric   }
470753f127fSDimitry Andric   IteratorGroup.wait();
471753f127fSDimitry Andric   std::unique_lock<std::mutex> Guard(IteratorMutex);
472753f127fSDimitry Andric   if (EC)
473753f127fSDimitry Andric     return errorCodeToError(EC);
474753f127fSDimitry Andric   return Error::success();
475753f127fSDimitry Andric }
476753f127fSDimitry Andric 
477bdd1243dSDimitry Andric Expected<std::optional<std::string>>
478753f127fSDimitry Andric DebuginfodCollection::getBinaryPath(BuildIDRef ID) {
479753f127fSDimitry Andric   Log.push("getting binary path of ID " + buildIDToString(ID));
480753f127fSDimitry Andric   std::shared_lock<sys::RWMutex> Guard(BinariesMutex);
481753f127fSDimitry Andric   auto Loc = Binaries.find(buildIDToString(ID));
482753f127fSDimitry Andric   if (Loc != Binaries.end()) {
483753f127fSDimitry Andric     std::string Path = Loc->getValue();
484753f127fSDimitry Andric     return Path;
485753f127fSDimitry Andric   }
486bdd1243dSDimitry Andric   return std::nullopt;
487753f127fSDimitry Andric }
488753f127fSDimitry Andric 
489bdd1243dSDimitry Andric Expected<std::optional<std::string>>
490753f127fSDimitry Andric DebuginfodCollection::getDebugBinaryPath(BuildIDRef ID) {
491753f127fSDimitry Andric   Log.push("getting debug binary path of ID " + buildIDToString(ID));
492753f127fSDimitry Andric   std::shared_lock<sys::RWMutex> Guard(DebugBinariesMutex);
493753f127fSDimitry Andric   auto Loc = DebugBinaries.find(buildIDToString(ID));
494753f127fSDimitry Andric   if (Loc != DebugBinaries.end()) {
495753f127fSDimitry Andric     std::string Path = Loc->getValue();
496753f127fSDimitry Andric     return Path;
497753f127fSDimitry Andric   }
498bdd1243dSDimitry Andric   return std::nullopt;
499753f127fSDimitry Andric }
500753f127fSDimitry Andric 
501753f127fSDimitry Andric Expected<std::string> DebuginfodCollection::findBinaryPath(BuildIDRef ID) {
502753f127fSDimitry Andric   {
503753f127fSDimitry Andric     // Check collection; perform on-demand update if stale.
504bdd1243dSDimitry Andric     Expected<std::optional<std::string>> PathOrErr = getBinaryPath(ID);
505753f127fSDimitry Andric     if (!PathOrErr)
506753f127fSDimitry Andric       return PathOrErr.takeError();
507bdd1243dSDimitry Andric     std::optional<std::string> Path = *PathOrErr;
508753f127fSDimitry Andric     if (!Path) {
509753f127fSDimitry Andric       Expected<bool> UpdatedOrErr = updateIfStale();
510753f127fSDimitry Andric       if (!UpdatedOrErr)
511753f127fSDimitry Andric         return UpdatedOrErr.takeError();
512753f127fSDimitry Andric       if (*UpdatedOrErr) {
513753f127fSDimitry Andric         // Try once more.
514753f127fSDimitry Andric         PathOrErr = getBinaryPath(ID);
515753f127fSDimitry Andric         if (!PathOrErr)
516753f127fSDimitry Andric           return PathOrErr.takeError();
517753f127fSDimitry Andric         Path = *PathOrErr;
518753f127fSDimitry Andric       }
519753f127fSDimitry Andric     }
520753f127fSDimitry Andric     if (Path)
521bdd1243dSDimitry Andric       return *Path;
522753f127fSDimitry Andric   }
523753f127fSDimitry Andric 
524753f127fSDimitry Andric   // Try federation.
525753f127fSDimitry Andric   Expected<std::string> PathOrErr = getCachedOrDownloadExecutable(ID);
526753f127fSDimitry Andric   if (!PathOrErr)
527753f127fSDimitry Andric     consumeError(PathOrErr.takeError());
528753f127fSDimitry Andric 
529753f127fSDimitry Andric   // Fall back to debug binary.
530753f127fSDimitry Andric   return findDebugBinaryPath(ID);
531753f127fSDimitry Andric }
532753f127fSDimitry Andric 
533753f127fSDimitry Andric Expected<std::string> DebuginfodCollection::findDebugBinaryPath(BuildIDRef ID) {
534753f127fSDimitry Andric   // Check collection; perform on-demand update if stale.
535bdd1243dSDimitry Andric   Expected<std::optional<std::string>> PathOrErr = getDebugBinaryPath(ID);
536753f127fSDimitry Andric   if (!PathOrErr)
537753f127fSDimitry Andric     return PathOrErr.takeError();
538bdd1243dSDimitry Andric   std::optional<std::string> Path = *PathOrErr;
539753f127fSDimitry Andric   if (!Path) {
540753f127fSDimitry Andric     Expected<bool> UpdatedOrErr = updateIfStale();
541753f127fSDimitry Andric     if (!UpdatedOrErr)
542753f127fSDimitry Andric       return UpdatedOrErr.takeError();
543753f127fSDimitry Andric     if (*UpdatedOrErr) {
544753f127fSDimitry Andric       // Try once more.
545753f127fSDimitry Andric       PathOrErr = getBinaryPath(ID);
546753f127fSDimitry Andric       if (!PathOrErr)
547753f127fSDimitry Andric         return PathOrErr.takeError();
548753f127fSDimitry Andric       Path = *PathOrErr;
549753f127fSDimitry Andric     }
550753f127fSDimitry Andric   }
551753f127fSDimitry Andric   if (Path)
552bdd1243dSDimitry Andric     return *Path;
553753f127fSDimitry Andric 
554753f127fSDimitry Andric   // Try federation.
555753f127fSDimitry Andric   return getCachedOrDownloadDebuginfo(ID);
556753f127fSDimitry Andric }
557753f127fSDimitry Andric 
558753f127fSDimitry Andric DebuginfodServer::DebuginfodServer(DebuginfodLog &Log,
559753f127fSDimitry Andric                                    DebuginfodCollection &Collection)
560753f127fSDimitry Andric     : Log(Log), Collection(Collection) {
561753f127fSDimitry Andric   cantFail(
562753f127fSDimitry Andric       Server.get(R"(/buildid/(.*)/debuginfo)", [&](HTTPServerRequest Request) {
563753f127fSDimitry Andric         Log.push("GET " + Request.UrlPath);
564753f127fSDimitry Andric         std::string IDString;
565753f127fSDimitry Andric         if (!tryGetFromHex(Request.UrlPathMatches[0], IDString)) {
566753f127fSDimitry Andric           Request.setResponse(
567753f127fSDimitry Andric               {404, "text/plain", "Build ID is not a hex string\n"});
568753f127fSDimitry Andric           return;
569753f127fSDimitry Andric         }
570bdd1243dSDimitry Andric         object::BuildID ID(IDString.begin(), IDString.end());
571753f127fSDimitry Andric         Expected<std::string> PathOrErr = Collection.findDebugBinaryPath(ID);
572753f127fSDimitry Andric         if (Error Err = PathOrErr.takeError()) {
573753f127fSDimitry Andric           consumeError(std::move(Err));
574753f127fSDimitry Andric           Request.setResponse({404, "text/plain", "Build ID not found\n"});
575753f127fSDimitry Andric           return;
576753f127fSDimitry Andric         }
577753f127fSDimitry Andric         streamFile(Request, *PathOrErr);
578753f127fSDimitry Andric       }));
579753f127fSDimitry Andric   cantFail(
580753f127fSDimitry Andric       Server.get(R"(/buildid/(.*)/executable)", [&](HTTPServerRequest Request) {
581753f127fSDimitry Andric         Log.push("GET " + Request.UrlPath);
582753f127fSDimitry Andric         std::string IDString;
583753f127fSDimitry Andric         if (!tryGetFromHex(Request.UrlPathMatches[0], IDString)) {
584753f127fSDimitry Andric           Request.setResponse(
585753f127fSDimitry Andric               {404, "text/plain", "Build ID is not a hex string\n"});
586753f127fSDimitry Andric           return;
587753f127fSDimitry Andric         }
588bdd1243dSDimitry Andric         object::BuildID ID(IDString.begin(), IDString.end());
589753f127fSDimitry Andric         Expected<std::string> PathOrErr = Collection.findBinaryPath(ID);
590753f127fSDimitry Andric         if (Error Err = PathOrErr.takeError()) {
591753f127fSDimitry Andric           consumeError(std::move(Err));
592753f127fSDimitry Andric           Request.setResponse({404, "text/plain", "Build ID not found\n"});
593753f127fSDimitry Andric           return;
594753f127fSDimitry Andric         }
595753f127fSDimitry Andric         streamFile(Request, *PathOrErr);
596753f127fSDimitry Andric       }));
597753f127fSDimitry Andric }
598753f127fSDimitry Andric 
5990eae32dcSDimitry Andric } // namespace llvm
600