1 //===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Token.h" 10 #include "clang/Basic/LangOptions.h" 11 #include "llvm/ADT/StringExtras.h" 12 #include "llvm/Support/Format.h" 13 #include "llvm/Support/FormatVariadic.h" 14 15 namespace clang { 16 namespace clangd { 17 18 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) { 19 OS << llvm::formatv("{0} {1}:{2} ", clang::tok::getTokenName(T.Kind), T.Line, 20 T.Indent); 21 OS << '"'; 22 llvm::printEscapedString(T.text(), OS); 23 OS << '"'; 24 if (T.Flags) 25 OS << llvm::format(" flags=%x", T.Flags); 26 return OS; 27 } 28 29 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) { 30 OS << "Index Kind Line Text\n"; 31 for (const auto &T : TS.tokens()) { 32 OS << llvm::format("%5d: %16s %4d:%-2d ", TS.index(T), 33 clang::tok::getTokenName(T.Kind), T.Line, T.Indent); 34 OS << '"'; 35 llvm::printEscapedString(T.text(), OS); 36 OS << '"'; 37 if (T.Flags) 38 OS << llvm::format(" flags=%x", T.Flags); 39 OS << '\n'; 40 } 41 return OS; 42 } 43 44 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) { 45 OS << llvm::formatv("[{0},{1})", R.Begin, R.End); 46 return OS; 47 } 48 49 TokenStream::TokenStream(std::shared_ptr<void> Payload) 50 : Payload(std::move(Payload)) { 51 Storage.emplace_back(); 52 Storage.back().Kind = clang::tok::eof; 53 } 54 55 void TokenStream::finalize() { 56 assert(!isFinalized()); 57 unsigned LastLine = Storage.back().Line; 58 Storage.emplace_back(); 59 Storage.back().Kind = tok::eof; 60 Storage.back().Line = LastLine + 1; 61 62 Tokens = Storage; 63 Tokens = Tokens.drop_front().drop_back(); 64 } 65 66 bool TokenStream::isFinalized() const { 67 assert(!Storage.empty() && Storage.front().Kind == tok::eof); 68 if (Storage.size() == 1) 69 return false; 70 return Storage.back().Kind == tok::eof; 71 } 72 73 void TokenStream::print(llvm::raw_ostream &OS) const { 74 bool FirstToken = true; 75 unsigned LastLine = -1; 76 StringRef LastText; 77 for (const auto &T : tokens()) { 78 StringRef Text = T.text(); 79 if (FirstToken) { 80 FirstToken = false; 81 } else if (T.Line == LastLine) { 82 if (LastText.data() + LastText.size() != Text.data()) 83 OS << ' '; 84 } else { 85 OS << '\n'; 86 OS.indent(T.Indent); 87 } 88 OS << Text; 89 LastLine = T.Line; 90 LastText = Text; 91 } 92 if (!FirstToken) 93 OS << '\n'; 94 } 95 96 clang::LangOptions genericLangOpts(clang::Language Lang, 97 clang::LangStandard::Kind Standard) { 98 clang::LangOptions Opts; 99 std::vector<std::string> UnusedIncludes; 100 LangOptions::setLangDefaults(Opts, Lang, llvm::Triple(), UnusedIncludes, 101 Standard); 102 103 // Some options are "on by default", but e.g. at the driver level. 104 if (Opts.CPlusPlus) 105 Opts.CXXOperatorNames = true; 106 if (Opts.CPlusPlus20) 107 Opts.Coroutines = true; 108 109 // Some options are off by default, but define keywords we want to tolerate. 110 if (Opts.CPlusPlus) 111 Opts.MicrosoftExt = true; // kw__try, kw__finally 112 Opts.DeclSpecKeyword = true; // __declspec 113 Opts.WChar = true; 114 115 return Opts; 116 } 117 118 TokenStream stripComments(const TokenStream &Input) { 119 TokenStream Out(Input.getPayload()); 120 for (const Token &T : Input.tokens()) { 121 if (T.Kind == tok::comment) 122 continue; 123 Out.push(T); 124 } 125 Out.finalize(); 126 return Out; 127 } 128 129 } // namespace clangd 130 } // namespace clang 131