1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "clang/Lex/Lexer.h"
11 #include "clang/Basic/Diagnostic.h"
12 #include "clang/Basic/DiagnosticOptions.h"
13 #include "clang/Basic/FileManager.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/TargetOptions.h"
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/HeaderSearchOptions.h"
20 #include "clang/Lex/ModuleLoader.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "gtest/gtest.h"
24
25 using namespace llvm;
26 using namespace clang;
27
28 namespace {
29
30 class VoidModuleLoader : public ModuleLoader {
loadModule(SourceLocation ImportLoc,ModuleIdPath Path,Module::NameVisibilityKind Visibility,bool IsInclusionDirective)31 ModuleLoadResult loadModule(SourceLocation ImportLoc,
32 ModuleIdPath Path,
33 Module::NameVisibilityKind Visibility,
34 bool IsInclusionDirective) override {
35 return ModuleLoadResult();
36 }
37
makeModuleVisible(Module * Mod,Module::NameVisibilityKind Visibility,SourceLocation ImportLoc,bool Complain)38 void makeModuleVisible(Module *Mod,
39 Module::NameVisibilityKind Visibility,
40 SourceLocation ImportLoc,
41 bool Complain) override { }
42
loadGlobalModuleIndex(SourceLocation TriggerLoc)43 GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override
44 { return nullptr; }
lookupMissingImports(StringRef Name,SourceLocation TriggerLoc)45 bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override
46 { return 0; };
47 };
48
49 // The test fixture.
50 class LexerTest : public ::testing::Test {
51 protected:
LexerTest()52 LexerTest()
53 : FileMgr(FileMgrOpts),
54 DiagID(new DiagnosticIDs()),
55 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
56 SourceMgr(Diags, FileMgr),
57 TargetOpts(new TargetOptions)
58 {
59 TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
60 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
61 }
62
CheckLex(StringRef Source,ArrayRef<tok::TokenKind> ExpectedTokens)63 std::vector<Token> CheckLex(StringRef Source,
64 ArrayRef<tok::TokenKind> ExpectedTokens) {
65 std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);
66 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
67
68 VoidModuleLoader ModLoader;
69 HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
70 Target.get());
71 Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr,
72 HeaderInfo, ModLoader, /*IILookup =*/nullptr,
73 /*OwnsHeaderSearch =*/false);
74 PP.Initialize(*Target);
75 PP.EnterMainSourceFile();
76
77 std::vector<Token> toks;
78 while (1) {
79 Token tok;
80 PP.Lex(tok);
81 if (tok.is(tok::eof))
82 break;
83 toks.push_back(tok);
84 }
85
86 EXPECT_EQ(ExpectedTokens.size(), toks.size());
87 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
88 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
89 }
90
91 return toks;
92 }
93
getSourceText(Token Begin,Token End)94 std::string getSourceText(Token Begin, Token End) {
95 bool Invalid;
96 StringRef Str =
97 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
98 Begin.getLocation(), End.getLocation())),
99 SourceMgr, LangOpts, &Invalid);
100 if (Invalid)
101 return "<INVALID>";
102 return Str;
103 }
104
105 FileSystemOptions FileMgrOpts;
106 FileManager FileMgr;
107 IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
108 DiagnosticsEngine Diags;
109 SourceManager SourceMgr;
110 LangOptions LangOpts;
111 std::shared_ptr<TargetOptions> TargetOpts;
112 IntrusiveRefCntPtr<TargetInfo> Target;
113 };
114
TEST_F(LexerTest,GetSourceTextExpandsToMaximumInMacroArgument)115 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
116 std::vector<tok::TokenKind> ExpectedTokens;
117 ExpectedTokens.push_back(tok::identifier);
118 ExpectedTokens.push_back(tok::l_paren);
119 ExpectedTokens.push_back(tok::identifier);
120 ExpectedTokens.push_back(tok::r_paren);
121
122 std::vector<Token> toks = CheckLex("#define M(x) x\n"
123 "M(f(M(i)))",
124 ExpectedTokens);
125
126 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
127 }
128
TEST_F(LexerTest,GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro)129 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
130 std::vector<tok::TokenKind> ExpectedTokens;
131 ExpectedTokens.push_back(tok::identifier);
132 ExpectedTokens.push_back(tok::identifier);
133
134 std::vector<Token> toks = CheckLex("#define M(x) x\n"
135 "M(M(i) c)",
136 ExpectedTokens);
137
138 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
139 }
140
TEST_F(LexerTest,GetSourceTextExpandsInMacroArgumentForBeginOfMacro)141 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
142 std::vector<tok::TokenKind> ExpectedTokens;
143 ExpectedTokens.push_back(tok::identifier);
144 ExpectedTokens.push_back(tok::identifier);
145 ExpectedTokens.push_back(tok::identifier);
146
147 std::vector<Token> toks = CheckLex("#define M(x) x\n"
148 "M(c c M(i))",
149 ExpectedTokens);
150
151 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
152 }
153
TEST_F(LexerTest,GetSourceTextExpandsInMacroArgumentForEndOfMacro)154 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
155 std::vector<tok::TokenKind> ExpectedTokens;
156 ExpectedTokens.push_back(tok::identifier);
157 ExpectedTokens.push_back(tok::identifier);
158 ExpectedTokens.push_back(tok::identifier);
159
160 std::vector<Token> toks = CheckLex("#define M(x) x\n"
161 "M(M(i) c c)",
162 ExpectedTokens);
163
164 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
165 }
166
TEST_F(LexerTest,GetSourceTextInSeparateFnMacros)167 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
168 std::vector<tok::TokenKind> ExpectedTokens;
169 ExpectedTokens.push_back(tok::identifier);
170 ExpectedTokens.push_back(tok::identifier);
171 ExpectedTokens.push_back(tok::identifier);
172 ExpectedTokens.push_back(tok::identifier);
173
174 std::vector<Token> toks = CheckLex("#define M(x) x\n"
175 "M(c M(i)) M(M(i) c)",
176 ExpectedTokens);
177
178 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
179 }
180
TEST_F(LexerTest,GetSourceTextWorksAcrossTokenPastes)181 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
182 std::vector<tok::TokenKind> ExpectedTokens;
183 ExpectedTokens.push_back(tok::identifier);
184 ExpectedTokens.push_back(tok::l_paren);
185 ExpectedTokens.push_back(tok::identifier);
186 ExpectedTokens.push_back(tok::r_paren);
187
188 std::vector<Token> toks = CheckLex("#define M(x) x\n"
189 "#define C(x) M(x##c)\n"
190 "M(f(C(i)))",
191 ExpectedTokens);
192
193 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
194 }
195
TEST_F(LexerTest,GetSourceTextExpandsAcrossMultipleMacroCalls)196 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
197 std::vector<tok::TokenKind> ExpectedTokens;
198 ExpectedTokens.push_back(tok::identifier);
199 ExpectedTokens.push_back(tok::l_paren);
200 ExpectedTokens.push_back(tok::identifier);
201 ExpectedTokens.push_back(tok::r_paren);
202
203 std::vector<Token> toks = CheckLex("#define M(x) x\n"
204 "f(M(M(i)))",
205 ExpectedTokens);
206 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
207 }
208
TEST_F(LexerTest,GetSourceTextInMiddleOfMacroArgument)209 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
210 std::vector<tok::TokenKind> ExpectedTokens;
211 ExpectedTokens.push_back(tok::identifier);
212 ExpectedTokens.push_back(tok::l_paren);
213 ExpectedTokens.push_back(tok::identifier);
214 ExpectedTokens.push_back(tok::r_paren);
215
216 std::vector<Token> toks = CheckLex("#define M(x) x\n"
217 "M(f(i))",
218 ExpectedTokens);
219 EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
220 }
221
TEST_F(LexerTest,GetSourceTextExpandsAroundDifferentMacroCalls)222 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
223 std::vector<tok::TokenKind> ExpectedTokens;
224 ExpectedTokens.push_back(tok::identifier);
225 ExpectedTokens.push_back(tok::l_paren);
226 ExpectedTokens.push_back(tok::identifier);
227 ExpectedTokens.push_back(tok::r_paren);
228
229 std::vector<Token> toks = CheckLex("#define M(x) x\n"
230 "#define C(x) x\n"
231 "f(C(M(i)))",
232 ExpectedTokens);
233 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
234 }
235
TEST_F(LexerTest,GetSourceTextOnlyExpandsIfFirstTokenInMacro)236 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
237 std::vector<tok::TokenKind> ExpectedTokens;
238 ExpectedTokens.push_back(tok::identifier);
239 ExpectedTokens.push_back(tok::l_paren);
240 ExpectedTokens.push_back(tok::identifier);
241 ExpectedTokens.push_back(tok::identifier);
242 ExpectedTokens.push_back(tok::r_paren);
243
244 std::vector<Token> toks = CheckLex("#define M(x) x\n"
245 "#define C(x) c x\n"
246 "f(C(M(i)))",
247 ExpectedTokens);
248 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
249 }
250
TEST_F(LexerTest,GetSourceTextExpandsRecursively)251 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
252 std::vector<tok::TokenKind> ExpectedTokens;
253 ExpectedTokens.push_back(tok::identifier);
254 ExpectedTokens.push_back(tok::identifier);
255 ExpectedTokens.push_back(tok::l_paren);
256 ExpectedTokens.push_back(tok::identifier);
257 ExpectedTokens.push_back(tok::r_paren);
258
259 std::vector<Token> toks = CheckLex("#define M(x) x\n"
260 "#define C(x) c M(x)\n"
261 "C(f(M(i)))",
262 ExpectedTokens);
263 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
264 }
265
TEST_F(LexerTest,LexAPI)266 TEST_F(LexerTest, LexAPI) {
267 std::vector<tok::TokenKind> ExpectedTokens;
268 ExpectedTokens.push_back(tok::l_square);
269 ExpectedTokens.push_back(tok::identifier);
270 ExpectedTokens.push_back(tok::r_square);
271 ExpectedTokens.push_back(tok::l_square);
272 ExpectedTokens.push_back(tok::identifier);
273 ExpectedTokens.push_back(tok::r_square);
274 ExpectedTokens.push_back(tok::identifier);
275 ExpectedTokens.push_back(tok::identifier);
276 ExpectedTokens.push_back(tok::identifier);
277 ExpectedTokens.push_back(tok::identifier);
278
279 std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
280 "#define N(x) x\n"
281 "#define INN(x) x\n"
282 "#define NOF1 INN(val)\n"
283 "#define NOF2 val\n"
284 "M(foo) N([bar])\n"
285 "N(INN(val)) N(NOF1) N(NOF2) N(val)",
286 ExpectedTokens);
287
288 SourceLocation lsqrLoc = toks[0].getLocation();
289 SourceLocation idLoc = toks[1].getLocation();
290 SourceLocation rsqrLoc = toks[2].getLocation();
291 std::pair<SourceLocation,SourceLocation>
292 macroPair = SourceMgr.getExpansionRange(lsqrLoc);
293 SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
294
295 SourceLocation Loc;
296 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
297 EXPECT_EQ(Loc, macroRange.getBegin());
298 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
299 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
300 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
301 EXPECT_EQ(Loc, macroRange.getEnd());
302
303 CharSourceRange range = Lexer::makeFileCharRange(
304 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
305 EXPECT_TRUE(range.isInvalid());
306 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
307 SourceMgr, LangOpts);
308 EXPECT_TRUE(range.isInvalid());
309 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
310 SourceMgr, LangOpts);
311 EXPECT_TRUE(!range.isTokenRange());
312 EXPECT_EQ(range.getAsRange(),
313 SourceRange(macroRange.getBegin(),
314 macroRange.getEnd().getLocWithOffset(1)));
315
316 StringRef text = Lexer::getSourceText(
317 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
318 SourceMgr, LangOpts);
319 EXPECT_EQ(text, "M(foo)");
320
321 SourceLocation macroLsqrLoc = toks[3].getLocation();
322 SourceLocation macroIdLoc = toks[4].getLocation();
323 SourceLocation macroRsqrLoc = toks[5].getLocation();
324 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
325 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
326 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
327
328 range = Lexer::makeFileCharRange(
329 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
330 SourceMgr, LangOpts);
331 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
332 range.getAsRange());
333
334 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
335 SourceMgr, LangOpts);
336 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
337 range.getAsRange());
338
339 macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
340 range = Lexer::makeFileCharRange(
341 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
342 SourceMgr, LangOpts);
343 EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
344 range.getAsRange());
345
346 text = Lexer::getSourceText(
347 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
348 SourceMgr, LangOpts);
349 EXPECT_EQ(text, "[bar");
350
351
352 SourceLocation idLoc1 = toks[6].getLocation();
353 SourceLocation idLoc2 = toks[7].getLocation();
354 SourceLocation idLoc3 = toks[8].getLocation();
355 SourceLocation idLoc4 = toks[9].getLocation();
356 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
357 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
358 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
359 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
360 }
361
362 } // anonymous namespace
363