1 //===-- BackgroundIndexLoader.cpp - ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "index/BackgroundIndexLoader.h" 10 #include "GlobalCompilationDatabase.h" 11 #include "Logger.h" 12 #include "Path.h" 13 #include "index/Background.h" 14 #include "llvm/ADT/DenseMap.h" 15 #include "llvm/ADT/DenseSet.h" 16 #include "llvm/ADT/SmallString.h" 17 #include "llvm/ADT/StringMap.h" 18 #include "llvm/Support/Path.h" 19 #include <string> 20 #include <utility> 21 #include <vector> 22 23 namespace clang { 24 namespace clangd { 25 namespace { 26 27 llvm::Optional<Path> uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) { 28 auto U = URI::parse(URI); 29 if (!U) 30 return llvm::None; 31 auto AbsolutePath = URI::resolve(*U, HintPath); 32 if (!AbsolutePath) 33 return llvm::None; 34 return *AbsolutePath; 35 } 36 37 /// A helper class to cache BackgroundIndexStorage operations and keep the 38 /// inverse dependency mapping. 39 class BackgroundIndexLoader { 40 public: 41 BackgroundIndexLoader(BackgroundIndexStorage::Factory &IndexStorageFactory) 42 : IndexStorageFactory(IndexStorageFactory) {} 43 /// Load the shards for \p MainFile and all of its dependencies. 44 void load(PathRef MainFile); 45 46 /// Consumes the loader and returns all shards. 47 std::vector<LoadedShard> takeResult() &&; 48 49 private: 50 /// Returns the Shard for \p StartSourceFile from cache or loads it from \p 51 /// Storage. Also returns paths for dependencies of \p StartSourceFile if it 52 /// wasn't cached yet. 53 std::pair<const LoadedShard &, std::vector<Path>> 54 loadShard(PathRef StartSourceFile); 55 56 /// Cache for Storage lookups. 57 llvm::StringMap<LoadedShard> LoadedShards; 58 59 /// References are into the AbsolutePaths in LoadedShards. 60 llvm::DenseMap<PathRef, PathRef> FileToTU; 61 62 BackgroundIndexStorage::Factory &IndexStorageFactory; 63 }; 64 65 std::pair<const LoadedShard &, std::vector<Path>> 66 BackgroundIndexLoader::loadShard(PathRef StartSourceFile) { 67 auto It = LoadedShards.try_emplace(StartSourceFile); 68 LoadedShard &LS = It.first->getValue(); 69 std::vector<Path> Edges = {}; 70 // Return the cached shard. 71 if (!It.second) 72 return {LS, Edges}; 73 74 LS.AbsolutePath = StartSourceFile.str(); 75 BackgroundIndexStorage *Storage = IndexStorageFactory(LS.AbsolutePath); 76 auto Shard = Storage->loadShard(StartSourceFile); 77 if (!Shard || !Shard->Sources) { 78 vlog("Failed to load shard: {0}", StartSourceFile); 79 return {LS, Edges}; 80 } 81 82 LS.Shard = std::move(Shard); 83 for (const auto &It : *LS.Shard->Sources) { 84 auto AbsPath = uriToAbsolutePath(It.getKey(), StartSourceFile); 85 if (!AbsPath) 86 continue; 87 // A shard contains only edges for non main-file sources. 88 if (*AbsPath != StartSourceFile) { 89 Edges.push_back(*AbsPath); 90 continue; 91 } 92 93 // Fill in shard metadata. 94 const IncludeGraphNode &IGN = It.getValue(); 95 LS.Digest = IGN.Digest; 96 LS.CountReferences = IGN.Flags & IncludeGraphNode::SourceFlag::IsTU; 97 LS.HadErrors = IGN.Flags & IncludeGraphNode::SourceFlag::HadErrors; 98 } 99 assert(LS.Digest != FileDigest{{0}} && "Digest is empty?"); 100 return {LS, Edges}; 101 } 102 103 void BackgroundIndexLoader::load(PathRef MainFile) { 104 llvm::StringSet<> InQueue; 105 // Following containers points to strings inside InQueue. 106 std::queue<PathRef> ToVisit; 107 InQueue.insert(MainFile); 108 ToVisit.push(MainFile); 109 110 while (!ToVisit.empty()) { 111 PathRef SourceFile = ToVisit.front(); 112 ToVisit.pop(); 113 114 auto ShardAndEdges = loadShard(SourceFile); 115 FileToTU[ShardAndEdges.first.AbsolutePath] = MainFile; 116 for (PathRef Edge : ShardAndEdges.second) { 117 auto It = InQueue.insert(Edge); 118 if (It.second) 119 ToVisit.push(It.first->getKey()); 120 } 121 } 122 } 123 124 std::vector<LoadedShard> BackgroundIndexLoader::takeResult() && { 125 std::vector<LoadedShard> Result; 126 Result.reserve(LoadedShards.size()); 127 for (auto &It : LoadedShards) { 128 Result.push_back(std::move(It.getValue())); 129 LoadedShard &LS = Result.back(); 130 auto TUsIt = FileToTU.find(LS.AbsolutePath); 131 assert(TUsIt != FileToTU.end() && "No TU registered for the shard"); 132 Result.back().DependentTU = TUsIt->second; 133 } 134 return Result; 135 } 136 } // namespace 137 138 std::vector<LoadedShard> 139 loadIndexShards(llvm::ArrayRef<Path> MainFiles, 140 BackgroundIndexStorage::Factory &IndexStorageFactory, 141 const GlobalCompilationDatabase &CDB) { 142 BackgroundIndexLoader Loader(IndexStorageFactory); 143 for (llvm::StringRef MainFile : MainFiles) { 144 assert(llvm::sys::path::is_absolute(MainFile)); 145 Loader.load(MainFile); 146 } 147 return std::move(Loader).takeResult(); 148 } 149 150 } // namespace clangd 151 } // namespace clang 152