1 //===--- TestLexer.h - Format C++ code --------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file contains a TestLexer to create FormatTokens from strings. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #ifndef CLANG_UNITTESTS_FORMAT_TESTLEXER_H 15 #define CLANG_UNITTESTS_FORMAT_TESTLEXER_H 16 17 #include "../../lib/Format/FormatTokenLexer.h" 18 #include "../../lib/Format/TokenAnalyzer.h" 19 #include "../../lib/Format/TokenAnnotator.h" 20 #include "../../lib/Format/UnwrappedLineParser.h" 21 22 #include "clang/Basic/FileManager.h" 23 #include "clang/Basic/SourceManager.h" 24 25 #include <numeric> 26 #include <ostream> 27 28 namespace clang { 29 namespace format { 30 31 typedef llvm::SmallVector<FormatToken *, 8> TokenList; 32 33 inline std::ostream &operator<<(std::ostream &Stream, const FormatToken &Tok) { 34 Stream << "(" << Tok.Tok.getName() << ", \"" << Tok.TokenText.str() << "\" , " 35 << getTokenTypeName(Tok.getType()) << ")"; 36 return Stream; 37 } 38 inline std::ostream &operator<<(std::ostream &Stream, const TokenList &Tokens) { 39 Stream << "{"; 40 for (size_t I = 0, E = Tokens.size(); I != E; ++I) 41 Stream << (I > 0 ? ", " : "") << *Tokens[I]; 42 Stream << "} (" << Tokens.size() << " tokens)"; 43 return Stream; 44 } 45 46 inline TokenList uneof(const TokenList &Tokens) { 47 assert(!Tokens.empty() && Tokens.back()->is(tok::eof)); 48 return TokenList(Tokens.begin(), std::prev(Tokens.end())); 49 } 50 51 inline std::string text(llvm::ArrayRef<FormatToken *> Tokens) { 52 return std::accumulate(Tokens.begin(), Tokens.end(), std::string(), 53 [](const std::string &R, FormatToken *Tok) { 54 return (R + Tok->TokenText).str(); 55 }); 56 } 57 58 class TestLexer : public UnwrappedLineConsumer { 59 public: 60 TestLexer(llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator, 61 std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Buffers, 62 FormatStyle Style = getLLVMStyle()) 63 : Allocator(Allocator), Buffers(Buffers), Style(Style), 64 SourceMgr("test.cpp", ""), IdentTable(getFormattingLangOpts(Style)) {} 65 66 TokenList lex(llvm::StringRef Code) { 67 FormatTokenLexer Lex = getNewLexer(Code); 68 ArrayRef<FormatToken *> Result = Lex.lex(); 69 return TokenList(Result.begin(), Result.end()); 70 } 71 72 TokenList annotate(llvm::StringRef Code) { 73 FormatTokenLexer Lex = getNewLexer(Code); 74 auto Tokens = Lex.lex(); 75 UnwrappedLineParser Parser(SourceMgr.get(), Style, Lex.getKeywords(), 0, 76 Tokens, *this, Allocator, IdentTable); 77 Parser.parse(); 78 TokenAnnotator Annotator(Style, Lex.getKeywords()); 79 for (auto &Line : UnwrappedLines) { 80 AnnotatedLine Annotated(Line); 81 Annotator.annotate(Annotated); 82 Annotator.calculateFormattingInformation(Annotated); 83 } 84 UnwrappedLines.clear(); 85 return TokenList(Tokens.begin(), Tokens.end()); 86 } 87 88 FormatToken *id(llvm::StringRef Code) { 89 auto Result = uneof(lex(Code)); 90 assert(Result.size() == 1U && "Code must expand to 1 token."); 91 return Result[0]; 92 } 93 94 protected: 95 void consumeUnwrappedLine(const UnwrappedLine &TheLine) override { 96 UnwrappedLines.push_back(TheLine); 97 } 98 void finishRun() override {} 99 100 FormatTokenLexer getNewLexer(StringRef Code) { 101 Buffers.push_back( 102 llvm::MemoryBuffer::getMemBufferCopy(Code, "<scratch space>")); 103 clang::FileID FID = 104 SourceMgr.get().createFileID(Buffers.back()->getMemBufferRef()); 105 return FormatTokenLexer(SourceMgr.get(), FID, 0, Style, Encoding, Allocator, 106 IdentTable); 107 } 108 109 public: 110 llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator; 111 std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Buffers; 112 FormatStyle Style; 113 encoding::Encoding Encoding = encoding::Encoding_UTF8; 114 clang::SourceManagerForFile SourceMgr; 115 IdentifierTable IdentTable; 116 SmallVector<UnwrappedLine, 16> UnwrappedLines; 117 }; 118 119 } // namespace format 120 } // namespace clang 121 122 #endif // LLVM_CLANG_UNITTESTS_FORMAT_TEST_LEXER_H 123