1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "clang/Lex/Lexer.h" 11 #include "clang/Basic/Diagnostic.h" 12 #include "clang/Basic/DiagnosticOptions.h" 13 #include "clang/Basic/FileManager.h" 14 #include "clang/Basic/LangOptions.h" 15 #include "clang/Basic/MemoryBufferCache.h" 16 #include "clang/Basic/SourceManager.h" 17 #include "clang/Basic/TargetInfo.h" 18 #include "clang/Basic/TargetOptions.h" 19 #include "clang/Lex/HeaderSearch.h" 20 #include "clang/Lex/HeaderSearchOptions.h" 21 #include "clang/Lex/ModuleLoader.h" 22 #include "clang/Lex/Preprocessor.h" 23 #include "clang/Lex/PreprocessorOptions.h" 24 #include "gtest/gtest.h" 25 26 using namespace clang; 27 28 namespace { 29 30 // The test fixture. 31 class LexerTest : public ::testing::Test { 32 protected: 33 LexerTest() 34 : FileMgr(FileMgrOpts), 35 DiagID(new DiagnosticIDs()), 36 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), 37 SourceMgr(Diags, FileMgr), 38 TargetOpts(new TargetOptions) 39 { 40 TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; 41 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); 42 } 43 44 std::vector<Token> Lex(StringRef Source) { 45 std::unique_ptr<llvm::MemoryBuffer> Buf = 46 llvm::MemoryBuffer::getMemBuffer(Source); 47 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf))); 48 49 TrivialModuleLoader ModLoader; 50 MemoryBufferCache PCMCache; 51 HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, 52 Diags, LangOpts, Target.get()); 53 Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts, 54 SourceMgr, PCMCache, HeaderInfo, ModLoader, 55 /*IILookup =*/nullptr, 56 /*OwnsHeaderSearch =*/false); 57 PP.Initialize(*Target); 58 PP.EnterMainSourceFile(); 59 60 std::vector<Token> toks; 61 while (1) { 62 Token tok; 63 PP.Lex(tok); 64 if (tok.is(tok::eof)) 65 break; 66 toks.push_back(tok); 67 } 68 69 return toks; 70 } 71 72 std::vector<Token> CheckLex(StringRef Source, 73 ArrayRef<tok::TokenKind> ExpectedTokens) { 74 auto toks = Lex(Source); 75 EXPECT_EQ(ExpectedTokens.size(), toks.size()); 76 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) { 77 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind()); 78 } 79 80 return toks; 81 } 82 83 std::string getSourceText(Token Begin, Token End) { 84 bool Invalid; 85 StringRef Str = 86 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange( 87 Begin.getLocation(), End.getLocation())), 88 SourceMgr, LangOpts, &Invalid); 89 if (Invalid) 90 return "<INVALID>"; 91 return Str; 92 } 93 94 FileSystemOptions FileMgrOpts; 95 FileManager FileMgr; 96 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 97 DiagnosticsEngine Diags; 98 SourceManager SourceMgr; 99 LangOptions LangOpts; 100 std::shared_ptr<TargetOptions> TargetOpts; 101 IntrusiveRefCntPtr<TargetInfo> Target; 102 }; 103 104 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) { 105 std::vector<tok::TokenKind> ExpectedTokens; 106 ExpectedTokens.push_back(tok::identifier); 107 ExpectedTokens.push_back(tok::l_paren); 108 ExpectedTokens.push_back(tok::identifier); 109 ExpectedTokens.push_back(tok::r_paren); 110 111 std::vector<Token> toks = CheckLex("#define M(x) x\n" 112 "M(f(M(i)))", 113 ExpectedTokens); 114 115 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2])); 116 } 117 118 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) { 119 std::vector<tok::TokenKind> ExpectedTokens; 120 ExpectedTokens.push_back(tok::identifier); 121 ExpectedTokens.push_back(tok::identifier); 122 123 std::vector<Token> toks = CheckLex("#define M(x) x\n" 124 "M(M(i) c)", 125 ExpectedTokens); 126 127 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0])); 128 } 129 130 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) { 131 std::vector<tok::TokenKind> ExpectedTokens; 132 ExpectedTokens.push_back(tok::identifier); 133 ExpectedTokens.push_back(tok::identifier); 134 ExpectedTokens.push_back(tok::identifier); 135 136 std::vector<Token> toks = CheckLex("#define M(x) x\n" 137 "M(c c M(i))", 138 ExpectedTokens); 139 140 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2])); 141 } 142 143 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) { 144 std::vector<tok::TokenKind> ExpectedTokens; 145 ExpectedTokens.push_back(tok::identifier); 146 ExpectedTokens.push_back(tok::identifier); 147 ExpectedTokens.push_back(tok::identifier); 148 149 std::vector<Token> toks = CheckLex("#define M(x) x\n" 150 "M(M(i) c c)", 151 ExpectedTokens); 152 153 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1])); 154 } 155 156 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) { 157 std::vector<tok::TokenKind> ExpectedTokens; 158 ExpectedTokens.push_back(tok::identifier); 159 ExpectedTokens.push_back(tok::identifier); 160 ExpectedTokens.push_back(tok::identifier); 161 ExpectedTokens.push_back(tok::identifier); 162 163 std::vector<Token> toks = CheckLex("#define M(x) x\n" 164 "M(c M(i)) M(M(i) c)", 165 ExpectedTokens); 166 167 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2])); 168 } 169 170 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) { 171 std::vector<tok::TokenKind> ExpectedTokens; 172 ExpectedTokens.push_back(tok::identifier); 173 ExpectedTokens.push_back(tok::l_paren); 174 ExpectedTokens.push_back(tok::identifier); 175 ExpectedTokens.push_back(tok::r_paren); 176 177 std::vector<Token> toks = CheckLex("#define M(x) x\n" 178 "#define C(x) M(x##c)\n" 179 "M(f(C(i)))", 180 ExpectedTokens); 181 182 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2])); 183 } 184 185 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) { 186 std::vector<tok::TokenKind> ExpectedTokens; 187 ExpectedTokens.push_back(tok::identifier); 188 ExpectedTokens.push_back(tok::l_paren); 189 ExpectedTokens.push_back(tok::identifier); 190 ExpectedTokens.push_back(tok::r_paren); 191 192 std::vector<Token> toks = CheckLex("#define M(x) x\n" 193 "f(M(M(i)))", 194 ExpectedTokens); 195 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2])); 196 } 197 198 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) { 199 std::vector<tok::TokenKind> ExpectedTokens; 200 ExpectedTokens.push_back(tok::identifier); 201 ExpectedTokens.push_back(tok::l_paren); 202 ExpectedTokens.push_back(tok::identifier); 203 ExpectedTokens.push_back(tok::r_paren); 204 205 std::vector<Token> toks = CheckLex("#define M(x) x\n" 206 "M(f(i))", 207 ExpectedTokens); 208 EXPECT_EQ("i", getSourceText(toks[2], toks[2])); 209 } 210 211 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) { 212 std::vector<tok::TokenKind> ExpectedTokens; 213 ExpectedTokens.push_back(tok::identifier); 214 ExpectedTokens.push_back(tok::l_paren); 215 ExpectedTokens.push_back(tok::identifier); 216 ExpectedTokens.push_back(tok::r_paren); 217 218 std::vector<Token> toks = CheckLex("#define M(x) x\n" 219 "#define C(x) x\n" 220 "f(C(M(i)))", 221 ExpectedTokens); 222 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2])); 223 } 224 225 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) { 226 std::vector<tok::TokenKind> ExpectedTokens; 227 ExpectedTokens.push_back(tok::identifier); 228 ExpectedTokens.push_back(tok::l_paren); 229 ExpectedTokens.push_back(tok::identifier); 230 ExpectedTokens.push_back(tok::identifier); 231 ExpectedTokens.push_back(tok::r_paren); 232 233 std::vector<Token> toks = CheckLex("#define M(x) x\n" 234 "#define C(x) c x\n" 235 "f(C(M(i)))", 236 ExpectedTokens); 237 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 238 } 239 240 TEST_F(LexerTest, GetSourceTextExpandsRecursively) { 241 std::vector<tok::TokenKind> ExpectedTokens; 242 ExpectedTokens.push_back(tok::identifier); 243 ExpectedTokens.push_back(tok::identifier); 244 ExpectedTokens.push_back(tok::l_paren); 245 ExpectedTokens.push_back(tok::identifier); 246 ExpectedTokens.push_back(tok::r_paren); 247 248 std::vector<Token> toks = CheckLex("#define M(x) x\n" 249 "#define C(x) c M(x)\n" 250 "C(f(M(i)))", 251 ExpectedTokens); 252 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 253 } 254 255 TEST_F(LexerTest, LexAPI) { 256 std::vector<tok::TokenKind> ExpectedTokens; 257 ExpectedTokens.push_back(tok::l_square); 258 ExpectedTokens.push_back(tok::identifier); 259 ExpectedTokens.push_back(tok::r_square); 260 ExpectedTokens.push_back(tok::l_square); 261 ExpectedTokens.push_back(tok::identifier); 262 ExpectedTokens.push_back(tok::r_square); 263 ExpectedTokens.push_back(tok::identifier); 264 ExpectedTokens.push_back(tok::identifier); 265 ExpectedTokens.push_back(tok::identifier); 266 ExpectedTokens.push_back(tok::identifier); 267 268 std::vector<Token> toks = CheckLex("#define M(x) [x]\n" 269 "#define N(x) x\n" 270 "#define INN(x) x\n" 271 "#define NOF1 INN(val)\n" 272 "#define NOF2 val\n" 273 "M(foo) N([bar])\n" 274 "N(INN(val)) N(NOF1) N(NOF2) N(val)", 275 ExpectedTokens); 276 277 SourceLocation lsqrLoc = toks[0].getLocation(); 278 SourceLocation idLoc = toks[1].getLocation(); 279 SourceLocation rsqrLoc = toks[2].getLocation(); 280 std::pair<SourceLocation,SourceLocation> 281 macroPair = SourceMgr.getExpansionRange(lsqrLoc); 282 SourceRange macroRange = SourceRange(macroPair.first, macroPair.second); 283 284 SourceLocation Loc; 285 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc)); 286 EXPECT_EQ(Loc, macroRange.getBegin()); 287 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 288 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 289 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc)); 290 EXPECT_EQ(Loc, macroRange.getEnd()); 291 292 CharSourceRange range = Lexer::makeFileCharRange( 293 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts); 294 EXPECT_TRUE(range.isInvalid()); 295 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc), 296 SourceMgr, LangOpts); 297 EXPECT_TRUE(range.isInvalid()); 298 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 299 SourceMgr, LangOpts); 300 EXPECT_TRUE(!range.isTokenRange()); 301 EXPECT_EQ(range.getAsRange(), 302 SourceRange(macroRange.getBegin(), 303 macroRange.getEnd().getLocWithOffset(1))); 304 305 StringRef text = Lexer::getSourceText( 306 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 307 SourceMgr, LangOpts); 308 EXPECT_EQ(text, "M(foo)"); 309 310 SourceLocation macroLsqrLoc = toks[3].getLocation(); 311 SourceLocation macroIdLoc = toks[4].getLocation(); 312 SourceLocation macroRsqrLoc = toks[5].getLocation(); 313 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc); 314 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc); 315 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc); 316 317 range = Lexer::makeFileCharRange( 318 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc), 319 SourceMgr, LangOpts); 320 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)), 321 range.getAsRange()); 322 323 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc), 324 SourceMgr, LangOpts); 325 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)), 326 range.getAsRange()); 327 328 macroPair = SourceMgr.getExpansionRange(macroLsqrLoc); 329 range = Lexer::makeFileCharRange( 330 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), 331 SourceMgr, LangOpts); 332 EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)), 333 range.getAsRange()); 334 335 text = Lexer::getSourceText( 336 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)), 337 SourceMgr, LangOpts); 338 EXPECT_EQ(text, "[bar"); 339 340 341 SourceLocation idLoc1 = toks[6].getLocation(); 342 SourceLocation idLoc2 = toks[7].getLocation(); 343 SourceLocation idLoc3 = toks[8].getLocation(); 344 SourceLocation idLoc4 = toks[9].getLocation(); 345 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts)); 346 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts)); 347 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts)); 348 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts)); 349 } 350 351 TEST_F(LexerTest, DontMergeMacroArgsFromDifferentMacroFiles) { 352 std::vector<Token> toks = 353 Lex("#define helper1 0\n" 354 "void helper2(const char *, ...);\n" 355 "#define M1(a, ...) helper2(a, ##__VA_ARGS__)\n" 356 "#define M2(a, ...) M1(a, helper1, ##__VA_ARGS__)\n" 357 "void f1() { M2(\"a\", \"b\"); }"); 358 359 // Check the file corresponding to the "helper1" macro arg in M2. 360 // 361 // The lexer used to report its size as 31, meaning that the end of the 362 // expansion would be on the *next line* (just past `M2("a", "b")`). Make 363 // sure that we get the correct end location (the comma after "helper1"). 364 SourceLocation helper1ArgLoc = toks[20].getLocation(); 365 EXPECT_EQ(SourceMgr.getFileIDSize(SourceMgr.getFileID(helper1ArgLoc)), 8U); 366 } 367 368 } // anonymous namespace 369