1 //===--- TestLexer.h - Format C++ code --------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file contains a TestLexer to create FormatTokens from strings.
11 ///
12 //===----------------------------------------------------------------------===//
13
14 #ifndef CLANG_UNITTESTS_FORMAT_TESTLEXER_H
15 #define CLANG_UNITTESTS_FORMAT_TESTLEXER_H
16
17 #include "../../lib/Format/FormatTokenLexer.h"
18 #include "../../lib/Format/TokenAnalyzer.h"
19 #include "../../lib/Format/TokenAnnotator.h"
20 #include "../../lib/Format/UnwrappedLineParser.h"
21
22 #include "clang/Basic/FileManager.h"
23 #include "clang/Basic/SourceManager.h"
24
25 #include <numeric>
26 #include <ostream>
27
28 namespace clang {
29 namespace format {
30
31 typedef SmallVector<FormatToken *, 8> TokenList;
32
33 inline std::ostream &operator<<(std::ostream &Stream, const FormatToken &Tok) {
34 Stream << "(" << Tok.Tok.getName() << ", \"" << Tok.TokenText.str() << "\" , "
35 << getTokenTypeName(Tok.getType()) << ")";
36 return Stream;
37 }
38 inline std::ostream &operator<<(std::ostream &Stream, const TokenList &Tokens) {
39 Stream << "{";
40 for (size_t I = 0, E = Tokens.size(); I != E; ++I)
41 Stream << (I > 0 ? ", " : "") << *Tokens[I];
42 Stream << "} (" << Tokens.size() << " tokens)";
43 return Stream;
44 }
45
uneof(const TokenList & Tokens)46 inline TokenList uneof(const TokenList &Tokens) {
47 assert(!Tokens.empty() && Tokens.back()->is(tok::eof));
48 return TokenList(Tokens.begin(), std::prev(Tokens.end()));
49 }
50
text(ArrayRef<FormatToken * > Tokens)51 inline std::string text(ArrayRef<FormatToken *> Tokens) {
52 return std::accumulate(Tokens.begin(), Tokens.end(), std::string(),
53 [](const std::string &R, FormatToken *Tok) {
54 return (R + Tok->TokenText).str();
55 });
56 }
57
58 class TestLexer : public UnwrappedLineConsumer {
59 public:
60 TestLexer(llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
61 std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Buffers,
62 FormatStyle Style = getLLVMStyle())
Allocator(Allocator)63 : Allocator(Allocator), Buffers(Buffers), Style(Style),
64 SourceMgr("test.cpp", ""), IdentTable(getFormattingLangOpts(Style)) {}
65
lex(StringRef Code)66 TokenList lex(StringRef Code) {
67 FormatTokenLexer Lex = getNewLexer(Code);
68 ArrayRef<FormatToken *> Result = Lex.lex();
69 return TokenList(Result.begin(), Result.end());
70 }
71
annotate(StringRef Code)72 TokenList annotate(StringRef Code) {
73 FormatTokenLexer Lex = getNewLexer(Code);
74 auto Tokens = Lex.lex();
75 UnwrappedLineParser Parser(SourceMgr.get(), Style, Lex.getKeywords(), 0,
76 Tokens, *this, Allocator, IdentTable);
77 Parser.parse();
78 TokenAnnotator Annotator(Style, Lex.getKeywords());
79 for (auto &Line : UnwrappedLines) {
80 AnnotatedLine Annotated(Line);
81 Annotator.annotate(Annotated);
82 Annotator.calculateFormattingInformation(Annotated);
83 }
84 UnwrappedLines.clear();
85 return TokenList(Tokens.begin(), Tokens.end());
86 }
87
id(StringRef Code)88 FormatToken *id(StringRef Code) {
89 auto Result = uneof(lex(Code));
90 assert(Result.size() == 1U && "Code must expand to 1 token.");
91 return Result[0];
92 }
93
94 protected:
consumeUnwrappedLine(const UnwrappedLine & TheLine)95 void consumeUnwrappedLine(const UnwrappedLine &TheLine) override {
96 UnwrappedLines.push_back(TheLine);
97 }
finishRun()98 void finishRun() override {}
99
getNewLexer(StringRef Code)100 FormatTokenLexer getNewLexer(StringRef Code) {
101 Buffers.push_back(
102 llvm::MemoryBuffer::getMemBufferCopy(Code, "<scratch space>"));
103 FileID FID =
104 SourceMgr.get().createFileID(Buffers.back()->getMemBufferRef());
105 return FormatTokenLexer(SourceMgr.get(), FID, 0, Style, Encoding, Allocator,
106 IdentTable);
107 }
108
109 public:
110 llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator;
111 std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Buffers;
112 FormatStyle Style;
113 encoding::Encoding Encoding = encoding::Encoding_UTF8;
114 SourceManagerForFile SourceMgr;
115 IdentifierTable IdentTable;
116 SmallVector<UnwrappedLine, 16> UnwrappedLines;
117 };
118
119 } // namespace format
120 } // namespace clang
121
122 #endif // LLVM_CLANG_UNITTESTS_FORMAT_TEST_LEXER_H
123