xref: /llvm-project/clang/unittests/Lex/LexerTest.cpp (revision 2d94bbb0c014880b67298aac8519cb60416feaf8) 
1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "clang/Lex/Lexer.h"
11 #include "clang/Basic/Diagnostic.h"
12 #include "clang/Basic/DiagnosticOptions.h"
13 #include "clang/Basic/FileManager.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/TargetOptions.h"
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/HeaderSearchOptions.h"
20 #include "clang/Lex/ModuleLoader.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "llvm/Config/config.h"
24 #include "gtest/gtest.h"
25 
26 using namespace llvm;
27 using namespace clang;
28 
29 namespace {
30 
31 class VoidModuleLoader : public ModuleLoader {
32   ModuleLoadResult loadModule(SourceLocation ImportLoc,
33                               ModuleIdPath Path,
34                               Module::NameVisibilityKind Visibility,
35                               bool IsInclusionDirective) override {
36     return ModuleLoadResult();
37   }
38 
39   void makeModuleVisible(Module *Mod,
40                          Module::NameVisibilityKind Visibility,
41                          SourceLocation ImportLoc,
42                          bool Complain) override { }
43 
44   GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override
45     { return 0; }
46   bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override
47     { return 0; };
48 };
49 
50 // The test fixture.
51 class LexerTest : public ::testing::Test {
52 protected:
53   LexerTest()
54     : FileMgr(FileMgrOpts),
55       DiagID(new DiagnosticIDs()),
56       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
57       SourceMgr(Diags, FileMgr),
58       TargetOpts(new TargetOptions)
59   {
60     TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
61     Target = TargetInfo::CreateTargetInfo(Diags, &*TargetOpts);
62   }
63 
64   std::vector<Token> CheckLex(StringRef Source,
65                               ArrayRef<tok::TokenKind> ExpectedTokens) {
66     MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source);
67     (void) SourceMgr.createMainFileIDForMemBuffer(buf);
68 
69     VoidModuleLoader ModLoader;
70     HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
71                             Target.getPtr());
72     Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, Target.getPtr(),
73                     SourceMgr, HeaderInfo, ModLoader, /*IILookup =*/ 0,
74                     /*OwnsHeaderSearch =*/ false,
75                     /*DelayInitialization =*/ false);
76     PP.EnterMainSourceFile();
77 
78     std::vector<Token> toks;
79     while (1) {
80       Token tok;
81       PP.Lex(tok);
82       if (tok.is(tok::eof))
83         break;
84       toks.push_back(tok);
85     }
86 
87     EXPECT_EQ(ExpectedTokens.size(), toks.size());
88     for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
89       EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
90     }
91 
92     return toks;
93   }
94 
95   std::string getSourceText(Token Begin, Token End) {
96     bool Invalid;
97     StringRef Str =
98         Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
99                                     Begin.getLocation(), End.getLocation())),
100                              SourceMgr, LangOpts, &Invalid);
101     if (Invalid)
102       return "<INVALID>";
103     return Str;
104   }
105 
106   FileSystemOptions FileMgrOpts;
107   FileManager FileMgr;
108   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
109   DiagnosticsEngine Diags;
110   SourceManager SourceMgr;
111   LangOptions LangOpts;
112   IntrusiveRefCntPtr<TargetOptions> TargetOpts;
113   IntrusiveRefCntPtr<TargetInfo> Target;
114 };
115 
116 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
117   std::vector<tok::TokenKind> ExpectedTokens;
118   ExpectedTokens.push_back(tok::identifier);
119   ExpectedTokens.push_back(tok::l_paren);
120   ExpectedTokens.push_back(tok::identifier);
121   ExpectedTokens.push_back(tok::r_paren);
122 
123   std::vector<Token> toks = CheckLex("#define M(x) x\n"
124                                      "M(f(M(i)))",
125                                      ExpectedTokens);
126 
127   EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
128 }
129 
130 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
131   std::vector<tok::TokenKind> ExpectedTokens;
132   ExpectedTokens.push_back(tok::identifier);
133   ExpectedTokens.push_back(tok::identifier);
134 
135   std::vector<Token> toks = CheckLex("#define M(x) x\n"
136                                      "M(M(i) c)",
137                                      ExpectedTokens);
138 
139   EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
140 }
141 
142 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
143   std::vector<tok::TokenKind> ExpectedTokens;
144   ExpectedTokens.push_back(tok::identifier);
145   ExpectedTokens.push_back(tok::identifier);
146   ExpectedTokens.push_back(tok::identifier);
147 
148   std::vector<Token> toks = CheckLex("#define M(x) x\n"
149                                      "M(c c M(i))",
150                                      ExpectedTokens);
151 
152   EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
153 }
154 
155 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
156   std::vector<tok::TokenKind> ExpectedTokens;
157   ExpectedTokens.push_back(tok::identifier);
158   ExpectedTokens.push_back(tok::identifier);
159   ExpectedTokens.push_back(tok::identifier);
160 
161   std::vector<Token> toks = CheckLex("#define M(x) x\n"
162                                      "M(M(i) c c)",
163                                      ExpectedTokens);
164 
165   EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
166 }
167 
168 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
169   std::vector<tok::TokenKind> ExpectedTokens;
170   ExpectedTokens.push_back(tok::identifier);
171   ExpectedTokens.push_back(tok::identifier);
172   ExpectedTokens.push_back(tok::identifier);
173   ExpectedTokens.push_back(tok::identifier);
174 
175   std::vector<Token> toks = CheckLex("#define M(x) x\n"
176                                      "M(c M(i)) M(M(i) c)",
177                                      ExpectedTokens);
178 
179   EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
180 }
181 
182 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
183   std::vector<tok::TokenKind> ExpectedTokens;
184   ExpectedTokens.push_back(tok::identifier);
185   ExpectedTokens.push_back(tok::l_paren);
186   ExpectedTokens.push_back(tok::identifier);
187   ExpectedTokens.push_back(tok::r_paren);
188 
189   std::vector<Token> toks = CheckLex("#define M(x) x\n"
190                                      "#define C(x) M(x##c)\n"
191                                      "M(f(C(i)))",
192                                      ExpectedTokens);
193 
194   EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
195 }
196 
197 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
198   std::vector<tok::TokenKind> ExpectedTokens;
199   ExpectedTokens.push_back(tok::identifier);
200   ExpectedTokens.push_back(tok::l_paren);
201   ExpectedTokens.push_back(tok::identifier);
202   ExpectedTokens.push_back(tok::r_paren);
203 
204   std::vector<Token> toks = CheckLex("#define M(x) x\n"
205                                      "f(M(M(i)))",
206                                      ExpectedTokens);
207   EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
208 }
209 
210 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
211   std::vector<tok::TokenKind> ExpectedTokens;
212   ExpectedTokens.push_back(tok::identifier);
213   ExpectedTokens.push_back(tok::l_paren);
214   ExpectedTokens.push_back(tok::identifier);
215   ExpectedTokens.push_back(tok::r_paren);
216 
217   std::vector<Token> toks = CheckLex("#define M(x) x\n"
218                                      "M(f(i))",
219                                      ExpectedTokens);
220   EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
221 }
222 
223 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
224   std::vector<tok::TokenKind> ExpectedTokens;
225   ExpectedTokens.push_back(tok::identifier);
226   ExpectedTokens.push_back(tok::l_paren);
227   ExpectedTokens.push_back(tok::identifier);
228   ExpectedTokens.push_back(tok::r_paren);
229 
230   std::vector<Token> toks = CheckLex("#define M(x) x\n"
231                                      "#define C(x) x\n"
232                                      "f(C(M(i)))",
233                                      ExpectedTokens);
234   EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
235 }
236 
237 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
238   std::vector<tok::TokenKind> ExpectedTokens;
239   ExpectedTokens.push_back(tok::identifier);
240   ExpectedTokens.push_back(tok::l_paren);
241   ExpectedTokens.push_back(tok::identifier);
242   ExpectedTokens.push_back(tok::identifier);
243   ExpectedTokens.push_back(tok::r_paren);
244 
245   std::vector<Token> toks = CheckLex("#define M(x) x\n"
246                                      "#define C(x) c x\n"
247                                      "f(C(M(i)))",
248                                      ExpectedTokens);
249   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
250 }
251 
252 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
253   std::vector<tok::TokenKind> ExpectedTokens;
254   ExpectedTokens.push_back(tok::identifier);
255   ExpectedTokens.push_back(tok::identifier);
256   ExpectedTokens.push_back(tok::l_paren);
257   ExpectedTokens.push_back(tok::identifier);
258   ExpectedTokens.push_back(tok::r_paren);
259 
260   std::vector<Token> toks = CheckLex("#define M(x) x\n"
261                                      "#define C(x) c M(x)\n"
262                                      "C(f(M(i)))",
263                                      ExpectedTokens);
264   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
265 }
266 
267 TEST_F(LexerTest, LexAPI) {
268   std::vector<tok::TokenKind> ExpectedTokens;
269   ExpectedTokens.push_back(tok::l_square);
270   ExpectedTokens.push_back(tok::identifier);
271   ExpectedTokens.push_back(tok::r_square);
272   ExpectedTokens.push_back(tok::l_square);
273   ExpectedTokens.push_back(tok::identifier);
274   ExpectedTokens.push_back(tok::r_square);
275   ExpectedTokens.push_back(tok::identifier);
276   ExpectedTokens.push_back(tok::identifier);
277   ExpectedTokens.push_back(tok::identifier);
278   ExpectedTokens.push_back(tok::identifier);
279 
280   std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
281                                      "#define N(x) x\n"
282                                      "#define INN(x) x\n"
283                                      "#define NOF1 INN(val)\n"
284                                      "#define NOF2 val\n"
285                                      "M(foo) N([bar])\n"
286                                      "N(INN(val)) N(NOF1) N(NOF2) N(val)",
287                                      ExpectedTokens);
288 
289   SourceLocation lsqrLoc = toks[0].getLocation();
290   SourceLocation idLoc = toks[1].getLocation();
291   SourceLocation rsqrLoc = toks[2].getLocation();
292   std::pair<SourceLocation,SourceLocation>
293     macroPair = SourceMgr.getExpansionRange(lsqrLoc);
294   SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
295 
296   SourceLocation Loc;
297   EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
298   EXPECT_EQ(Loc, macroRange.getBegin());
299   EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
300   EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
301   EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
302   EXPECT_EQ(Loc, macroRange.getEnd());
303 
304   CharSourceRange range = Lexer::makeFileCharRange(
305            CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
306   EXPECT_TRUE(range.isInvalid());
307   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
308                                    SourceMgr, LangOpts);
309   EXPECT_TRUE(range.isInvalid());
310   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
311                                    SourceMgr, LangOpts);
312   EXPECT_TRUE(!range.isTokenRange());
313   EXPECT_EQ(range.getAsRange(),
314             SourceRange(macroRange.getBegin(),
315                         macroRange.getEnd().getLocWithOffset(1)));
316 
317   StringRef text = Lexer::getSourceText(
318                                CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
319                                SourceMgr, LangOpts);
320   EXPECT_EQ(text, "M(foo)");
321 
322   SourceLocation macroLsqrLoc = toks[3].getLocation();
323   SourceLocation macroIdLoc = toks[4].getLocation();
324   SourceLocation macroRsqrLoc = toks[5].getLocation();
325   SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
326   SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
327   SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
328 
329   range = Lexer::makeFileCharRange(
330       CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
331       SourceMgr, LangOpts);
332   EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
333             range.getAsRange());
334 
335   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
336                                    SourceMgr, LangOpts);
337   EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
338             range.getAsRange());
339 
340   macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
341   range = Lexer::makeFileCharRange(
342                      CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
343                      SourceMgr, LangOpts);
344   EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
345             range.getAsRange());
346 
347   text = Lexer::getSourceText(
348           CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
349           SourceMgr, LangOpts);
350   EXPECT_EQ(text, "[bar");
351 
352 
353   SourceLocation idLoc1 = toks[6].getLocation();
354   SourceLocation idLoc2 = toks[7].getLocation();
355   SourceLocation idLoc3 = toks[8].getLocation();
356   SourceLocation idLoc4 = toks[9].getLocation();
357   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
358   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
359   EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
360   EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
361 }
362 
363 } // anonymous namespace
364