xref: /llvm-project/clang/unittests/Lex/LexerTest.cpp (revision 1f76c4e8101b9beaf8f1b10a57faa80256ab2b05) 
1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "clang/Lex/Lexer.h"
11 #include "clang/Basic/Diagnostic.h"
12 #include "clang/Basic/DiagnosticOptions.h"
13 #include "clang/Basic/FileManager.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/TargetOptions.h"
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/HeaderSearchOptions.h"
20 #include "clang/Lex/ModuleLoader.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "llvm/Config/config.h"
24 #include "gtest/gtest.h"
25 
26 using namespace llvm;
27 using namespace clang;
28 
29 namespace {
30 
31 class VoidModuleLoader : public ModuleLoader {
32   virtual ModuleLoadResult loadModule(SourceLocation ImportLoc,
33                                       ModuleIdPath Path,
34                                       Module::NameVisibilityKind Visibility,
35                                       bool IsInclusionDirective) {
36     return ModuleLoadResult();
37   }
38 
39   virtual void makeModuleVisible(Module *Mod,
40                                  Module::NameVisibilityKind Visibility,
41                                  SourceLocation ImportLoc,
42                                  bool Complain) { }
43 };
44 
45 // The test fixture.
46 class LexerTest : public ::testing::Test {
47 protected:
48   LexerTest()
49     : FileMgr(FileMgrOpts),
50       DiagID(new DiagnosticIDs()),
51       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
52       SourceMgr(Diags, FileMgr),
53       TargetOpts(new TargetOptions)
54   {
55     TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
56     Target = TargetInfo::CreateTargetInfo(Diags, &*TargetOpts);
57   }
58 
59   std::vector<Token> CheckLex(StringRef Source,
60                               ArrayRef<tok::TokenKind> ExpectedTokens) {
61     MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source);
62     (void) SourceMgr.createMainFileIDForMemBuffer(buf);
63 
64     VoidModuleLoader ModLoader;
65     HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
66                             Target.getPtr());
67     Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, Target.getPtr(),
68                     SourceMgr, HeaderInfo, ModLoader, /*IILookup =*/ 0,
69                     /*OwnsHeaderSearch =*/ false,
70                     /*DelayInitialization =*/ false);
71     PP.EnterMainSourceFile();
72 
73     std::vector<Token> toks;
74     while (1) {
75       Token tok;
76       PP.Lex(tok);
77       if (tok.is(tok::eof))
78         break;
79       toks.push_back(tok);
80     }
81 
82     EXPECT_EQ(ExpectedTokens.size(), toks.size());
83     for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
84       EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
85     }
86 
87     return toks;
88   }
89 
90   std::string getSourceText(Token Begin, Token End) {
91     bool Invalid;
92     StringRef Str =
93         Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
94                                     Begin.getLocation(), End.getLocation())),
95                              SourceMgr, LangOpts, &Invalid);
96     if (Invalid)
97       return "<INVALID>";
98     return Str;
99   }
100 
101   FileSystemOptions FileMgrOpts;
102   FileManager FileMgr;
103   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
104   DiagnosticsEngine Diags;
105   SourceManager SourceMgr;
106   LangOptions LangOpts;
107   IntrusiveRefCntPtr<TargetOptions> TargetOpts;
108   IntrusiveRefCntPtr<TargetInfo> Target;
109 };
110 
111 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
112   std::vector<tok::TokenKind> ExpectedTokens;
113   ExpectedTokens.push_back(tok::identifier);
114   ExpectedTokens.push_back(tok::l_paren);
115   ExpectedTokens.push_back(tok::identifier);
116   ExpectedTokens.push_back(tok::r_paren);
117 
118   std::vector<Token> toks = CheckLex("#define M(x) x\n"
119                                      "M(f(M(i)))",
120                                      ExpectedTokens);
121 
122   EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
123 }
124 
125 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
126   std::vector<tok::TokenKind> ExpectedTokens;
127   ExpectedTokens.push_back(tok::identifier);
128   ExpectedTokens.push_back(tok::identifier);
129 
130   std::vector<Token> toks = CheckLex("#define M(x) x\n"
131                                      "M(M(i) c)",
132                                      ExpectedTokens);
133 
134   EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
135 }
136 
137 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
138   std::vector<tok::TokenKind> ExpectedTokens;
139   ExpectedTokens.push_back(tok::identifier);
140   ExpectedTokens.push_back(tok::identifier);
141   ExpectedTokens.push_back(tok::identifier);
142 
143   std::vector<Token> toks = CheckLex("#define M(x) x\n"
144                                      "M(c c M(i))",
145                                      ExpectedTokens);
146 
147   EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
148 }
149 
150 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
151   std::vector<tok::TokenKind> ExpectedTokens;
152   ExpectedTokens.push_back(tok::identifier);
153   ExpectedTokens.push_back(tok::identifier);
154   ExpectedTokens.push_back(tok::identifier);
155 
156   std::vector<Token> toks = CheckLex("#define M(x) x\n"
157                                      "M(M(i) c c)",
158                                      ExpectedTokens);
159 
160   EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
161 }
162 
163 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
164   std::vector<tok::TokenKind> ExpectedTokens;
165   ExpectedTokens.push_back(tok::identifier);
166   ExpectedTokens.push_back(tok::identifier);
167   ExpectedTokens.push_back(tok::identifier);
168   ExpectedTokens.push_back(tok::identifier);
169 
170   std::vector<Token> toks = CheckLex("#define M(x) x\n"
171                                      "M(c M(i)) M(M(i) c)",
172                                      ExpectedTokens);
173 
174   EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
175 }
176 
177 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
178   std::vector<tok::TokenKind> ExpectedTokens;
179   ExpectedTokens.push_back(tok::identifier);
180   ExpectedTokens.push_back(tok::l_paren);
181   ExpectedTokens.push_back(tok::identifier);
182   ExpectedTokens.push_back(tok::r_paren);
183 
184   std::vector<Token> toks = CheckLex("#define M(x) x\n"
185                                      "#define C(x) M(x##c)\n"
186                                      "M(f(C(i)))",
187                                      ExpectedTokens);
188 
189   EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
190 }
191 
192 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
193   std::vector<tok::TokenKind> ExpectedTokens;
194   ExpectedTokens.push_back(tok::identifier);
195   ExpectedTokens.push_back(tok::l_paren);
196   ExpectedTokens.push_back(tok::identifier);
197   ExpectedTokens.push_back(tok::r_paren);
198 
199   std::vector<Token> toks = CheckLex("#define M(x) x\n"
200                                      "f(M(M(i)))",
201                                      ExpectedTokens);
202   EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
203 }
204 
205 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
206   std::vector<tok::TokenKind> ExpectedTokens;
207   ExpectedTokens.push_back(tok::identifier);
208   ExpectedTokens.push_back(tok::l_paren);
209   ExpectedTokens.push_back(tok::identifier);
210   ExpectedTokens.push_back(tok::r_paren);
211 
212   std::vector<Token> toks = CheckLex("#define M(x) x\n"
213                                      "M(f(i))",
214                                      ExpectedTokens);
215   EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
216 }
217 
218 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
219   std::vector<tok::TokenKind> ExpectedTokens;
220   ExpectedTokens.push_back(tok::identifier);
221   ExpectedTokens.push_back(tok::l_paren);
222   ExpectedTokens.push_back(tok::identifier);
223   ExpectedTokens.push_back(tok::r_paren);
224 
225   std::vector<Token> toks = CheckLex("#define M(x) x\n"
226                                      "#define C(x) x\n"
227                                      "f(C(M(i)))",
228                                      ExpectedTokens);
229   EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
230 }
231 
232 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
233   std::vector<tok::TokenKind> ExpectedTokens;
234   ExpectedTokens.push_back(tok::identifier);
235   ExpectedTokens.push_back(tok::l_paren);
236   ExpectedTokens.push_back(tok::identifier);
237   ExpectedTokens.push_back(tok::identifier);
238   ExpectedTokens.push_back(tok::r_paren);
239 
240   std::vector<Token> toks = CheckLex("#define M(x) x\n"
241                                      "#define C(x) c x\n"
242                                      "f(C(M(i)))",
243                                      ExpectedTokens);
244   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
245 }
246 
247 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
248   std::vector<tok::TokenKind> ExpectedTokens;
249   ExpectedTokens.push_back(tok::identifier);
250   ExpectedTokens.push_back(tok::identifier);
251   ExpectedTokens.push_back(tok::l_paren);
252   ExpectedTokens.push_back(tok::identifier);
253   ExpectedTokens.push_back(tok::r_paren);
254 
255   std::vector<Token> toks = CheckLex("#define M(x) x\n"
256                                      "#define C(x) c M(x)\n"
257                                      "C(f(M(i)))",
258                                      ExpectedTokens);
259   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
260 }
261 
262 TEST_F(LexerTest, LexAPI) {
263   std::vector<tok::TokenKind> ExpectedTokens;
264   ExpectedTokens.push_back(tok::l_square);
265   ExpectedTokens.push_back(tok::identifier);
266   ExpectedTokens.push_back(tok::r_square);
267   ExpectedTokens.push_back(tok::l_square);
268   ExpectedTokens.push_back(tok::identifier);
269   ExpectedTokens.push_back(tok::r_square);
270   ExpectedTokens.push_back(tok::identifier);
271   ExpectedTokens.push_back(tok::identifier);
272   ExpectedTokens.push_back(tok::identifier);
273   ExpectedTokens.push_back(tok::identifier);
274 
275   std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
276                                      "#define N(x) x\n"
277                                      "#define INN(x) x\n"
278                                      "#define NOF1 INN(val)\n"
279                                      "#define NOF2 val\n"
280                                      "M(foo) N([bar])\n"
281                                      "N(INN(val)) N(NOF1) N(NOF2) N(val)",
282                                      ExpectedTokens);
283 
284   SourceLocation lsqrLoc = toks[0].getLocation();
285   SourceLocation idLoc = toks[1].getLocation();
286   SourceLocation rsqrLoc = toks[2].getLocation();
287   std::pair<SourceLocation,SourceLocation>
288     macroPair = SourceMgr.getExpansionRange(lsqrLoc);
289   SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
290 
291   SourceLocation Loc;
292   EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
293   EXPECT_EQ(Loc, macroRange.getBegin());
294   EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
295   EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
296   EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
297   EXPECT_EQ(Loc, macroRange.getEnd());
298 
299   CharSourceRange range = Lexer::makeFileCharRange(
300            CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
301   EXPECT_TRUE(range.isInvalid());
302   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
303                                    SourceMgr, LangOpts);
304   EXPECT_TRUE(range.isInvalid());
305   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
306                                    SourceMgr, LangOpts);
307   EXPECT_TRUE(!range.isTokenRange());
308   EXPECT_EQ(range.getAsRange(),
309             SourceRange(macroRange.getBegin(),
310                         macroRange.getEnd().getLocWithOffset(1)));
311 
312   StringRef text = Lexer::getSourceText(
313                                CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
314                                SourceMgr, LangOpts);
315   EXPECT_EQ(text, "M(foo)");
316 
317   SourceLocation macroLsqrLoc = toks[3].getLocation();
318   SourceLocation macroIdLoc = toks[4].getLocation();
319   SourceLocation macroRsqrLoc = toks[5].getLocation();
320   SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
321   SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
322   SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
323 
324   range = Lexer::makeFileCharRange(
325       CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
326       SourceMgr, LangOpts);
327   EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
328             range.getAsRange());
329 
330   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
331                                    SourceMgr, LangOpts);
332   EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
333             range.getAsRange());
334 
335   macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
336   range = Lexer::makeFileCharRange(
337                      CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
338                      SourceMgr, LangOpts);
339   EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
340             range.getAsRange());
341 
342   text = Lexer::getSourceText(
343           CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
344           SourceMgr, LangOpts);
345   EXPECT_EQ(text, "[bar");
346 
347 
348   SourceLocation idLoc1 = toks[6].getLocation();
349   SourceLocation idLoc2 = toks[7].getLocation();
350   SourceLocation idLoc3 = toks[8].getLocation();
351   SourceLocation idLoc4 = toks[9].getLocation();
352   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
353   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
354   EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
355   EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
356 }
357 
358 } // anonymous namespace
359