1 //===--- JSONCompilationDatabase.cpp - ------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the implementation of the JSONCompilationDatabase. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "clang/Tooling/JSONCompilationDatabase.h" 15 #include "clang/Tooling/CompilationDatabase.h" 16 #include "clang/Tooling/CompilationDatabasePluginRegistry.h" 17 #include "clang/Tooling/Tooling.h" 18 #include "llvm/ADT/SmallString.h" 19 #include "llvm/Support/Allocator.h" 20 #include "llvm/Support/CommandLine.h" 21 #include "llvm/Support/Path.h" 22 #include "llvm/Support/StringSaver.h" 23 #include <system_error> 24 25 namespace clang { 26 namespace tooling { 27 28 namespace { 29 30 /// \brief A parser for escaped strings of command line arguments. 31 /// 32 /// Assumes \-escaping for quoted arguments (see the documentation of 33 /// unescapeCommandLine(...)). 34 class CommandLineArgumentParser { 35 public: 36 CommandLineArgumentParser(StringRef CommandLine) 37 : Input(CommandLine), Position(Input.begin()-1) {} 38 39 std::vector<std::string> parse() { 40 bool HasMoreInput = true; 41 while (HasMoreInput && nextNonWhitespace()) { 42 std::string Argument; 43 HasMoreInput = parseStringInto(Argument); 44 CommandLine.push_back(Argument); 45 } 46 return CommandLine; 47 } 48 49 private: 50 // All private methods return true if there is more input available. 51 52 bool parseStringInto(std::string &String) { 53 do { 54 if (*Position == '"') { 55 if (!parseDoubleQuotedStringInto(String)) return false; 56 } else if (*Position == '\'') { 57 if (!parseSingleQuotedStringInto(String)) return false; 58 } else { 59 if (!parseFreeStringInto(String)) return false; 60 } 61 } while (*Position != ' '); 62 return true; 63 } 64 65 bool parseDoubleQuotedStringInto(std::string &String) { 66 if (!next()) return false; 67 while (*Position != '"') { 68 if (!skipEscapeCharacter()) return false; 69 String.push_back(*Position); 70 if (!next()) return false; 71 } 72 return next(); 73 } 74 75 bool parseSingleQuotedStringInto(std::string &String) { 76 if (!next()) return false; 77 while (*Position != '\'') { 78 String.push_back(*Position); 79 if (!next()) return false; 80 } 81 return next(); 82 } 83 84 bool parseFreeStringInto(std::string &String) { 85 do { 86 if (!skipEscapeCharacter()) return false; 87 String.push_back(*Position); 88 if (!next()) return false; 89 } while (*Position != ' ' && *Position != '"' && *Position != '\''); 90 return true; 91 } 92 93 bool skipEscapeCharacter() { 94 if (*Position == '\\') { 95 return next(); 96 } 97 return true; 98 } 99 100 bool nextNonWhitespace() { 101 do { 102 if (!next()) return false; 103 } while (*Position == ' '); 104 return true; 105 } 106 107 bool next() { 108 ++Position; 109 return Position != Input.end(); 110 } 111 112 const StringRef Input; 113 StringRef::iterator Position; 114 std::vector<std::string> CommandLine; 115 }; 116 117 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax, 118 StringRef EscapedCommandLine) { 119 if (Syntax == JSONCommandLineSyntax::AutoDetect) { 120 Syntax = JSONCommandLineSyntax::Gnu; 121 llvm::Triple Triple(llvm::sys::getProcessTriple()); 122 if (Triple.getOS() == llvm::Triple::OSType::Win32) { 123 // Assume Windows command line parsing on Win32 unless the triple 124 // explicitly tells us otherwise. 125 if (!Triple.hasEnvironment() || 126 Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC) 127 Syntax = JSONCommandLineSyntax::Windows; 128 } 129 } 130 131 if (Syntax == JSONCommandLineSyntax::Windows) { 132 llvm::BumpPtrAllocator Alloc; 133 llvm::StringSaver Saver(Alloc); 134 llvm::SmallVector<const char *, 64> T; 135 llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T); 136 std::vector<std::string> Result(T.begin(), T.end()); 137 return Result; 138 } 139 assert(Syntax == JSONCommandLineSyntax::Gnu); 140 CommandLineArgumentParser parser(EscapedCommandLine); 141 return parser.parse(); 142 } 143 144 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin { 145 std::unique_ptr<CompilationDatabase> 146 loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override { 147 SmallString<1024> JSONDatabasePath(Directory); 148 llvm::sys::path::append(JSONDatabasePath, "compile_commands.json"); 149 return JSONCompilationDatabase::loadFromFile( 150 JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect); 151 } 152 }; 153 154 } // end namespace 155 156 // Register the JSONCompilationDatabasePlugin with the 157 // CompilationDatabasePluginRegistry using this statically initialized variable. 158 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin> 159 X("json-compilation-database", "Reads JSON formatted compilation databases"); 160 161 // This anchor is used to force the linker to link in the generated object file 162 // and thus register the JSONCompilationDatabasePlugin. 163 volatile int JSONAnchorSource = 0; 164 165 std::unique_ptr<JSONCompilationDatabase> 166 JSONCompilationDatabase::loadFromFile(StringRef FilePath, 167 std::string &ErrorMessage, 168 JSONCommandLineSyntax Syntax) { 169 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer = 170 llvm::MemoryBuffer::getFile(FilePath); 171 if (std::error_code Result = DatabaseBuffer.getError()) { 172 ErrorMessage = "Error while opening JSON database: " + Result.message(); 173 return nullptr; 174 } 175 std::unique_ptr<JSONCompilationDatabase> Database( 176 new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax)); 177 if (!Database->parse(ErrorMessage)) 178 return nullptr; 179 return Database; 180 } 181 182 std::unique_ptr<JSONCompilationDatabase> 183 JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString, 184 std::string &ErrorMessage, 185 JSONCommandLineSyntax Syntax) { 186 std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer( 187 llvm::MemoryBuffer::getMemBuffer(DatabaseString)); 188 std::unique_ptr<JSONCompilationDatabase> Database( 189 new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax)); 190 if (!Database->parse(ErrorMessage)) 191 return nullptr; 192 return Database; 193 } 194 195 std::vector<CompileCommand> 196 JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const { 197 SmallString<128> NativeFilePath; 198 llvm::sys::path::native(FilePath, NativeFilePath); 199 200 std::string Error; 201 llvm::raw_string_ostream ES(Error); 202 StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES); 203 if (Match.empty()) 204 return std::vector<CompileCommand>(); 205 llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator 206 CommandsRefI = IndexByFile.find(Match); 207 if (CommandsRefI == IndexByFile.end()) 208 return std::vector<CompileCommand>(); 209 std::vector<CompileCommand> Commands; 210 getCommands(CommandsRefI->getValue(), Commands); 211 return Commands; 212 } 213 214 std::vector<std::string> 215 JSONCompilationDatabase::getAllFiles() const { 216 std::vector<std::string> Result; 217 218 llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator 219 CommandsRefI = IndexByFile.begin(); 220 const llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator 221 CommandsRefEnd = IndexByFile.end(); 222 for (; CommandsRefI != CommandsRefEnd; ++CommandsRefI) { 223 Result.push_back(CommandsRefI->first().str()); 224 } 225 226 return Result; 227 } 228 229 std::vector<CompileCommand> 230 JSONCompilationDatabase::getAllCompileCommands() const { 231 std::vector<CompileCommand> Commands; 232 getCommands(AllCommands, Commands); 233 return Commands; 234 } 235 236 static std::vector<std::string> 237 nodeToCommandLine(JSONCommandLineSyntax Syntax, 238 const std::vector<llvm::yaml::ScalarNode *> &Nodes) { 239 SmallString<1024> Storage; 240 if (Nodes.size() == 1) { 241 return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage)); 242 } 243 std::vector<std::string> Arguments; 244 for (auto *Node : Nodes) { 245 Arguments.push_back(Node->getValue(Storage)); 246 } 247 return Arguments; 248 } 249 250 void JSONCompilationDatabase::getCommands( 251 ArrayRef<CompileCommandRef> CommandsRef, 252 std::vector<CompileCommand> &Commands) const { 253 for (int I = 0, E = CommandsRef.size(); I != E; ++I) { 254 SmallString<8> DirectoryStorage; 255 SmallString<32> FilenameStorage; 256 SmallString<32> OutputStorage; 257 auto Output = std::get<3>(CommandsRef[I]); 258 Commands.emplace_back( 259 std::get<0>(CommandsRef[I])->getValue(DirectoryStorage), 260 std::get<1>(CommandsRef[I])->getValue(FilenameStorage), 261 nodeToCommandLine(Syntax, std::get<2>(CommandsRef[I])), 262 Output ? Output->getValue(OutputStorage) : ""); 263 } 264 } 265 266 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) { 267 llvm::yaml::document_iterator I = YAMLStream.begin(); 268 if (I == YAMLStream.end()) { 269 ErrorMessage = "Error while parsing YAML."; 270 return false; 271 } 272 llvm::yaml::Node *Root = I->getRoot(); 273 if (!Root) { 274 ErrorMessage = "Error while parsing YAML."; 275 return false; 276 } 277 llvm::yaml::SequenceNode *Array = dyn_cast<llvm::yaml::SequenceNode>(Root); 278 if (!Array) { 279 ErrorMessage = "Expected array."; 280 return false; 281 } 282 for (auto& NextObject : *Array) { 283 llvm::yaml::MappingNode *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject); 284 if (!Object) { 285 ErrorMessage = "Expected object."; 286 return false; 287 } 288 llvm::yaml::ScalarNode *Directory = nullptr; 289 llvm::Optional<std::vector<llvm::yaml::ScalarNode *>> Command; 290 llvm::yaml::ScalarNode *File = nullptr; 291 llvm::yaml::ScalarNode *Output = nullptr; 292 for (auto& NextKeyValue : *Object) { 293 llvm::yaml::ScalarNode *KeyString = 294 dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey()); 295 if (!KeyString) { 296 ErrorMessage = "Expected strings as key."; 297 return false; 298 } 299 SmallString<10> KeyStorage; 300 StringRef KeyValue = KeyString->getValue(KeyStorage); 301 llvm::yaml::Node *Value = NextKeyValue.getValue(); 302 if (!Value) { 303 ErrorMessage = "Expected value."; 304 return false; 305 } 306 llvm::yaml::ScalarNode *ValueString = 307 dyn_cast<llvm::yaml::ScalarNode>(Value); 308 llvm::yaml::SequenceNode *SequenceString = 309 dyn_cast<llvm::yaml::SequenceNode>(Value); 310 if (KeyValue == "arguments" && !SequenceString) { 311 ErrorMessage = "Expected sequence as value."; 312 return false; 313 } else if (KeyValue != "arguments" && !ValueString) { 314 ErrorMessage = "Expected string as value."; 315 return false; 316 } 317 if (KeyValue == "directory") { 318 Directory = ValueString; 319 } else if (KeyValue == "arguments") { 320 Command = std::vector<llvm::yaml::ScalarNode *>(); 321 for (auto &Argument : *SequenceString) { 322 auto Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument); 323 if (!Scalar) { 324 ErrorMessage = "Only strings are allowed in 'arguments'."; 325 return false; 326 } 327 Command->push_back(Scalar); 328 } 329 } else if (KeyValue == "command") { 330 if (!Command) 331 Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString); 332 } else if (KeyValue == "file") { 333 File = ValueString; 334 } else if (KeyValue == "output") { 335 Output = ValueString; 336 } else { 337 ErrorMessage = ("Unknown key: \"" + 338 KeyString->getRawValue() + "\"").str(); 339 return false; 340 } 341 } 342 if (!File) { 343 ErrorMessage = "Missing key: \"file\"."; 344 return false; 345 } 346 if (!Command) { 347 ErrorMessage = "Missing key: \"command\" or \"arguments\"."; 348 return false; 349 } 350 if (!Directory) { 351 ErrorMessage = "Missing key: \"directory\"."; 352 return false; 353 } 354 SmallString<8> FileStorage; 355 StringRef FileName = File->getValue(FileStorage); 356 SmallString<128> NativeFilePath; 357 if (llvm::sys::path::is_relative(FileName)) { 358 SmallString<8> DirectoryStorage; 359 SmallString<128> AbsolutePath( 360 Directory->getValue(DirectoryStorage)); 361 llvm::sys::path::append(AbsolutePath, FileName); 362 llvm::sys::path::native(AbsolutePath, NativeFilePath); 363 } else { 364 llvm::sys::path::native(FileName, NativeFilePath); 365 } 366 auto Cmd = CompileCommandRef(Directory, File, *Command, Output); 367 IndexByFile[NativeFilePath].push_back(Cmd); 368 AllCommands.push_back(Cmd); 369 MatchTrie.insert(NativeFilePath); 370 } 371 return true; 372 } 373 374 } // end namespace tooling 375 } // end namespace clang 376