1 //===-- ClangDocMain.cpp - ClangDoc -----------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This tool for generating C and C++ documentation from source code 10 // and comments. Generally, it runs a LibTooling FrontendAction on source files, 11 // mapping each declaration in those files to its USR and serializing relevant 12 // information into LLVM bitcode. It then runs a pass over the collected 13 // declaration information, reducing by USR. There is an option to dump this 14 // intermediate result to bitcode. Finally, it hands the reduced information 15 // off to a generator, which does the final parsing from the intermediate 16 // representation to the desired output format. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "BitcodeReader.h" 21 #include "BitcodeWriter.h" 22 #include "ClangDoc.h" 23 #include "Generators.h" 24 #include "Representation.h" 25 #include "clang/AST/AST.h" 26 #include "clang/AST/Decl.h" 27 #include "clang/ASTMatchers/ASTMatchFinder.h" 28 #include "clang/ASTMatchers/ASTMatchersInternal.h" 29 #include "clang/Driver/Options.h" 30 #include "clang/Frontend/FrontendActions.h" 31 #include "clang/Tooling/AllTUsExecution.h" 32 #include "clang/Tooling/CommonOptionsParser.h" 33 #include "clang/Tooling/Execution.h" 34 #include "clang/Tooling/Tooling.h" 35 #include "llvm/ADT/APFloat.h" 36 #include "llvm/Support/CommandLine.h" 37 #include "llvm/Support/Error.h" 38 #include "llvm/Support/FileSystem.h" 39 #include "llvm/Support/Mutex.h" 40 #include "llvm/Support/Path.h" 41 #include "llvm/Support/Process.h" 42 #include "llvm/Support/Signals.h" 43 #include "llvm/Support/ThreadPool.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include <atomic> 46 #include <mutex> 47 #include <string> 48 49 using namespace clang::ast_matchers; 50 using namespace clang::tooling; 51 using namespace clang; 52 53 static llvm::cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage); 54 static llvm::cl::OptionCategory ClangDocCategory("clang-doc options"); 55 56 static llvm::cl::opt<std::string> 57 ProjectName("project-name", llvm::cl::desc("Name of project."), 58 llvm::cl::cat(ClangDocCategory)); 59 60 static llvm::cl::opt<bool> IgnoreMappingFailures( 61 "ignore-map-errors", 62 llvm::cl::desc("Continue if files are not mapped correctly."), 63 llvm::cl::init(true), llvm::cl::cat(ClangDocCategory)); 64 65 static llvm::cl::opt<std::string> 66 OutDirectory("output", 67 llvm::cl::desc("Directory for outputting generated files."), 68 llvm::cl::init("docs"), llvm::cl::cat(ClangDocCategory)); 69 70 static llvm::cl::opt<bool> 71 PublicOnly("public", llvm::cl::desc("Document only public declarations."), 72 llvm::cl::init(false), llvm::cl::cat(ClangDocCategory)); 73 74 static llvm::cl::opt<bool> DoxygenOnly( 75 "doxygen", 76 llvm::cl::desc("Use only doxygen-style comments to generate docs."), 77 llvm::cl::init(false), llvm::cl::cat(ClangDocCategory)); 78 79 static llvm::cl::list<std::string> UserStylesheets( 80 "stylesheets", llvm::cl::CommaSeparated, 81 llvm::cl::desc("CSS stylesheets to extend the default styles."), 82 llvm::cl::cat(ClangDocCategory)); 83 84 static llvm::cl::opt<std::string> UserAssetPath( 85 "asset", 86 llvm::cl::desc("User supplied asset path to " 87 "override the default css and js files for html output"), 88 llvm::cl::cat(ClangDocCategory)); 89 90 static llvm::cl::opt<std::string> SourceRoot("source-root", llvm::cl::desc(R"( 91 Directory where processed files are stored. 92 Links to definition locations will only be 93 generated if the file is in this dir.)"), 94 llvm::cl::cat(ClangDocCategory)); 95 96 static llvm::cl::opt<std::string> 97 RepositoryUrl("repository", llvm::cl::desc(R"( 98 URL of repository that hosts code. 99 Used for links to definition locations.)"), 100 llvm::cl::cat(ClangDocCategory)); 101 102 enum OutputFormatTy { 103 md, 104 yaml, 105 html, 106 }; 107 108 static llvm::cl::opt<OutputFormatTy> 109 FormatEnum("format", llvm::cl::desc("Format for outputted docs."), 110 llvm::cl::values(clEnumValN(OutputFormatTy::yaml, "yaml", 111 "Documentation in YAML format."), 112 clEnumValN(OutputFormatTy::md, "md", 113 "Documentation in MD format."), 114 clEnumValN(OutputFormatTy::html, "html", 115 "Documentation in HTML format.")), 116 llvm::cl::init(OutputFormatTy::yaml), 117 llvm::cl::cat(ClangDocCategory)); 118 119 std::string getFormatString() { 120 switch (FormatEnum) { 121 case OutputFormatTy::yaml: 122 return "yaml"; 123 case OutputFormatTy::md: 124 return "md"; 125 case OutputFormatTy::html: 126 return "html"; 127 } 128 llvm_unreachable("Unknown OutputFormatTy"); 129 } 130 131 // This function isn't referenced outside its translation unit, but it 132 // can't use the "static" keyword because its address is used for 133 // GetMainExecutable (since some platforms don't support taking the 134 // address of main, and some platforms can't implement GetMainExecutable 135 // without being given the address of a function in the main executable). 136 std::string getExecutablePath(const char *Argv0, void *MainAddr) { 137 return llvm::sys::fs::getMainExecutable(Argv0, MainAddr); 138 } 139 140 llvm::Error getAssetFiles(clang::doc::ClangDocContext &CDCtx) { 141 using DirIt = llvm::sys::fs::directory_iterator; 142 std::error_code FileErr; 143 llvm::SmallString<128> FilePath(UserAssetPath); 144 for (DirIt DirStart = DirIt(UserAssetPath, FileErr), 145 DirEnd; 146 !FileErr && DirStart != DirEnd; DirStart.increment(FileErr)) { 147 FilePath = DirStart->path(); 148 if (llvm::sys::fs::is_regular_file(FilePath)) { 149 if (llvm::sys::path::extension(FilePath) == ".css") 150 CDCtx.UserStylesheets.insert(CDCtx.UserStylesheets.begin(), 151 std::string(FilePath)); 152 else if (llvm::sys::path::extension(FilePath) == ".js") 153 CDCtx.JsScripts.emplace_back(FilePath.str()); 154 } 155 } 156 if (FileErr) 157 return llvm::createFileError(FilePath, FileErr); 158 return llvm::Error::success(); 159 } 160 161 llvm::Error getDefaultAssetFiles(const char *Argv0, 162 clang::doc::ClangDocContext &CDCtx) { 163 void *MainAddr = (void *)(intptr_t)getExecutablePath; 164 std::string ClangDocPath = getExecutablePath(Argv0, MainAddr); 165 llvm::SmallString<128> NativeClangDocPath; 166 llvm::sys::path::native(ClangDocPath, NativeClangDocPath); 167 168 llvm::SmallString<128> AssetsPath; 169 AssetsPath = llvm::sys::path::parent_path(NativeClangDocPath); 170 llvm::sys::path::append(AssetsPath, "..", "share", "clang-doc"); 171 llvm::SmallString<128> DefaultStylesheet; 172 llvm::sys::path::native(AssetsPath, DefaultStylesheet); 173 llvm::sys::path::append(DefaultStylesheet, 174 "clang-doc-default-stylesheet.css"); 175 llvm::SmallString<128> IndexJS; 176 llvm::sys::path::native(AssetsPath, IndexJS); 177 llvm::sys::path::append(IndexJS, "index.js"); 178 179 if (!llvm::sys::fs::is_regular_file(IndexJS)) 180 return llvm::createStringError(llvm::inconvertibleErrorCode(), 181 "default index.js file missing at " + 182 IndexJS + "\n"); 183 184 if (!llvm::sys::fs::is_regular_file(DefaultStylesheet)) 185 return llvm::createStringError( 186 llvm::inconvertibleErrorCode(), 187 "default clang-doc-default-stylesheet.css file missing at " + 188 DefaultStylesheet + "\n"); 189 190 CDCtx.UserStylesheets.insert(CDCtx.UserStylesheets.begin(), 191 std::string(DefaultStylesheet)); 192 CDCtx.JsScripts.emplace_back(IndexJS.str()); 193 194 return llvm::Error::success(); 195 } 196 197 llvm::Error getHtmlAssetFiles(const char *Argv0, 198 clang::doc::ClangDocContext &CDCtx) { 199 if (!UserAssetPath.empty() && 200 !llvm::sys::fs::is_directory(std::string(UserAssetPath))) 201 llvm::outs() << "Asset path supply is not a directory: " << UserAssetPath 202 << " falling back to default\n"; 203 if (llvm::sys::fs::is_directory(std::string(UserAssetPath))) 204 return getAssetFiles(CDCtx); 205 return getDefaultAssetFiles(Argv0, CDCtx); 206 } 207 208 /// Make the output of clang-doc deterministic by sorting the children of 209 /// namespaces and records. 210 void sortUsrToInfo(llvm::StringMap<std::unique_ptr<doc::Info>> &USRToInfo) { 211 for (auto &I : USRToInfo) { 212 auto &Info = I.second; 213 if (Info->IT == doc::InfoType::IT_namespace) { 214 auto *Namespace = static_cast<clang::doc::NamespaceInfo *>(Info.get()); 215 Namespace->Children.sort(); 216 } 217 if (Info->IT == doc::InfoType::IT_record) { 218 auto *Record = static_cast<clang::doc::RecordInfo *>(Info.get()); 219 Record->Children.sort(); 220 } 221 } 222 } 223 224 int main(int argc, const char **argv) { 225 llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); 226 std::error_code OK; 227 228 const char *Overview = 229 R"(Generates documentation from source code and comments. 230 231 Example usage for files without flags (default): 232 233 $ clang-doc File1.cpp File2.cpp ... FileN.cpp 234 235 Example usage for a project using a compile commands database: 236 237 $ clang-doc --executor=all-TUs compile_commands.json 238 )"; 239 240 auto Executor = clang::tooling::createExecutorFromCommandLineArgs( 241 argc, argv, ClangDocCategory, Overview); 242 243 if (!Executor) { 244 llvm::errs() << toString(Executor.takeError()) << "\n"; 245 return 1; 246 } 247 248 // Fail early if an invalid format was provided. 249 std::string Format = getFormatString(); 250 llvm::outs() << "Emiting docs in " << Format << " format.\n"; 251 auto G = doc::findGeneratorByName(Format); 252 if (!G) { 253 llvm::errs() << toString(G.takeError()) << "\n"; 254 return 1; 255 } 256 257 ArgumentsAdjuster ArgAdjuster; 258 if (!DoxygenOnly) 259 ArgAdjuster = combineAdjusters( 260 getInsertArgumentAdjuster("-fparse-all-comments", 261 tooling::ArgumentInsertPosition::END), 262 ArgAdjuster); 263 264 clang::doc::ClangDocContext CDCtx = { 265 Executor->get()->getExecutionContext(), 266 ProjectName, 267 PublicOnly, 268 OutDirectory, 269 SourceRoot, 270 RepositoryUrl, 271 {UserStylesheets.begin(), UserStylesheets.end()} 272 }; 273 274 if (Format == "html") { 275 if (auto Err = getHtmlAssetFiles(argv[0], CDCtx)) { 276 llvm::errs() << toString(std::move(Err)) << "\n"; 277 return 1; 278 } 279 } 280 281 // Mapping phase 282 llvm::outs() << "Mapping decls...\n"; 283 auto Err = 284 Executor->get()->execute(doc::newMapperActionFactory(CDCtx), ArgAdjuster); 285 if (Err) { 286 if (IgnoreMappingFailures) 287 llvm::errs() << "Error mapping decls in files. Clang-doc will ignore " 288 "these files and continue:\n" 289 << toString(std::move(Err)) << "\n"; 290 else { 291 llvm::errs() << toString(std::move(Err)) << "\n"; 292 return 1; 293 } 294 } 295 296 // Collect values into output by key. 297 // In ToolResults, the Key is the hashed USR and the value is the 298 // bitcode-encoded representation of the Info object. 299 llvm::outs() << "Collecting infos...\n"; 300 llvm::StringMap<std::vector<StringRef>> USRToBitcode; 301 Executor->get()->getToolResults()->forEachResult( 302 [&](StringRef Key, StringRef Value) { 303 USRToBitcode[Key].emplace_back(Value); 304 }); 305 306 // Collects all Infos according to their unique USR value. This map is added 307 // to from the thread pool below and is protected by the USRToInfoMutex. 308 llvm::sys::Mutex USRToInfoMutex; 309 llvm::StringMap<std::unique_ptr<doc::Info>> USRToInfo; 310 311 // First reducing phase (reduce all decls into one info per decl). 312 llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n"; 313 std::atomic<bool> Error; 314 Error = false; 315 llvm::sys::Mutex IndexMutex; 316 // ExecutorConcurrency is a flag exposed by AllTUsExecution.h 317 llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ExecutorConcurrency)); 318 for (auto &Group : USRToBitcode) { 319 Pool.async([&]() { 320 std::vector<std::unique_ptr<doc::Info>> Infos; 321 for (auto &Bitcode : Group.getValue()) { 322 llvm::BitstreamCursor Stream(Bitcode); 323 doc::ClangDocBitcodeReader Reader(Stream); 324 auto ReadInfos = Reader.readBitcode(); 325 if (!ReadInfos) { 326 llvm::errs() << toString(ReadInfos.takeError()) << "\n"; 327 Error = true; 328 return; 329 } 330 std::move(ReadInfos->begin(), ReadInfos->end(), 331 std::back_inserter(Infos)); 332 } 333 334 auto Reduced = doc::mergeInfos(Infos); 335 if (!Reduced) { 336 llvm::errs() << llvm::toString(Reduced.takeError()); 337 return; 338 } 339 340 // Add a reference to this Info in the Index 341 { 342 std::lock_guard<llvm::sys::Mutex> Guard(IndexMutex); 343 clang::doc::Generator::addInfoToIndex(CDCtx.Idx, Reduced.get().get()); 344 } 345 346 // Save in the result map (needs a lock due to threaded access). 347 { 348 std::lock_guard<llvm::sys::Mutex> Guard(USRToInfoMutex); 349 USRToInfo[Group.getKey()] = std::move(Reduced.get()); 350 } 351 }); 352 } 353 354 Pool.wait(); 355 356 if (Error) 357 return 1; 358 359 sortUsrToInfo(USRToInfo); 360 361 // Ensure the root output directory exists. 362 if (std::error_code Err = llvm::sys::fs::create_directories(OutDirectory); 363 Err != std::error_code()) { 364 llvm::errs() << "Failed to create directory '" << OutDirectory << "'\n"; 365 return 1; 366 } 367 368 // Run the generator. 369 llvm::outs() << "Generating docs...\n"; 370 if (auto Err = 371 G->get()->generateDocs(OutDirectory, std::move(USRToInfo), CDCtx)) { 372 llvm::errs() << toString(std::move(Err)) << "\n"; 373 return 1; 374 } 375 376 llvm::outs() << "Generating assets for docs...\n"; 377 Err = G->get()->createResources(CDCtx); 378 if (Err) { 379 llvm::outs() << "warning: " << toString(std::move(Err)) << "\n"; 380 } 381 382 return 0; 383 } 384