1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Lex/Lexer.h" 10 #include "clang/Basic/Diagnostic.h" 11 #include "clang/Basic/DiagnosticOptions.h" 12 #include "clang/Basic/FileManager.h" 13 #include "clang/Basic/LangOptions.h" 14 #include "clang/Basic/SourceLocation.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "clang/Basic/TargetInfo.h" 17 #include "clang/Basic/TargetOptions.h" 18 #include "clang/Basic/TokenKinds.h" 19 #include "clang/Lex/HeaderSearch.h" 20 #include "clang/Lex/HeaderSearchOptions.h" 21 #include "clang/Lex/MacroArgs.h" 22 #include "clang/Lex/MacroInfo.h" 23 #include "clang/Lex/ModuleLoader.h" 24 #include "clang/Lex/Preprocessor.h" 25 #include "clang/Lex/PreprocessorOptions.h" 26 #include "gmock/gmock.h" 27 #include "gtest/gtest.h" 28 #include <vector> 29 30 namespace { 31 using namespace clang; 32 using testing::ElementsAre; 33 34 // The test fixture. 35 class LexerTest : public ::testing::Test { 36 protected: 37 LexerTest() 38 : FileMgr(FileMgrOpts), 39 DiagID(new DiagnosticIDs()), 40 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), 41 SourceMgr(Diags, FileMgr), 42 TargetOpts(new TargetOptions) 43 { 44 TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; 45 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); 46 } 47 48 std::unique_ptr<Preprocessor> CreatePP(StringRef Source, 49 TrivialModuleLoader &ModLoader) { 50 std::unique_ptr<llvm::MemoryBuffer> Buf = 51 llvm::MemoryBuffer::getMemBuffer(Source); 52 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf))); 53 54 HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, 55 Diags, LangOpts, Target.get()); 56 std::unique_ptr<Preprocessor> PP = std::make_unique<Preprocessor>( 57 std::make_shared<PreprocessorOptions>(), Diags, LangOpts, SourceMgr, 58 HeaderInfo, ModLoader, 59 /*IILookup =*/nullptr, 60 /*OwnsHeaderSearch =*/false); 61 PP->Initialize(*Target); 62 PP->EnterMainSourceFile(); 63 return PP; 64 } 65 66 std::vector<Token> Lex(StringRef Source) { 67 TrivialModuleLoader ModLoader; 68 auto PP = CreatePP(Source, ModLoader); 69 70 std::vector<Token> toks; 71 while (1) { 72 Token tok; 73 PP->Lex(tok); 74 if (tok.is(tok::eof)) 75 break; 76 toks.push_back(tok); 77 } 78 79 return toks; 80 } 81 82 std::vector<Token> CheckLex(StringRef Source, 83 ArrayRef<tok::TokenKind> ExpectedTokens) { 84 auto toks = Lex(Source); 85 EXPECT_EQ(ExpectedTokens.size(), toks.size()); 86 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) { 87 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind()); 88 } 89 90 return toks; 91 } 92 93 std::string getSourceText(Token Begin, Token End) { 94 bool Invalid; 95 StringRef Str = 96 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange( 97 Begin.getLocation(), End.getLocation())), 98 SourceMgr, LangOpts, &Invalid); 99 if (Invalid) 100 return "<INVALID>"; 101 return Str; 102 } 103 104 FileSystemOptions FileMgrOpts; 105 FileManager FileMgr; 106 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 107 DiagnosticsEngine Diags; 108 SourceManager SourceMgr; 109 LangOptions LangOpts; 110 std::shared_ptr<TargetOptions> TargetOpts; 111 IntrusiveRefCntPtr<TargetInfo> Target; 112 }; 113 114 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) { 115 std::vector<tok::TokenKind> ExpectedTokens; 116 ExpectedTokens.push_back(tok::identifier); 117 ExpectedTokens.push_back(tok::l_paren); 118 ExpectedTokens.push_back(tok::identifier); 119 ExpectedTokens.push_back(tok::r_paren); 120 121 std::vector<Token> toks = CheckLex("#define M(x) x\n" 122 "M(f(M(i)))", 123 ExpectedTokens); 124 125 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2])); 126 } 127 128 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) { 129 std::vector<tok::TokenKind> ExpectedTokens; 130 ExpectedTokens.push_back(tok::identifier); 131 ExpectedTokens.push_back(tok::identifier); 132 133 std::vector<Token> toks = CheckLex("#define M(x) x\n" 134 "M(M(i) c)", 135 ExpectedTokens); 136 137 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0])); 138 } 139 140 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) { 141 std::vector<tok::TokenKind> ExpectedTokens; 142 ExpectedTokens.push_back(tok::identifier); 143 ExpectedTokens.push_back(tok::identifier); 144 ExpectedTokens.push_back(tok::identifier); 145 146 std::vector<Token> toks = CheckLex("#define M(x) x\n" 147 "M(c c M(i))", 148 ExpectedTokens); 149 150 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2])); 151 } 152 153 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) { 154 std::vector<tok::TokenKind> ExpectedTokens; 155 ExpectedTokens.push_back(tok::identifier); 156 ExpectedTokens.push_back(tok::identifier); 157 ExpectedTokens.push_back(tok::identifier); 158 159 std::vector<Token> toks = CheckLex("#define M(x) x\n" 160 "M(M(i) c c)", 161 ExpectedTokens); 162 163 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1])); 164 } 165 166 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) { 167 std::vector<tok::TokenKind> ExpectedTokens; 168 ExpectedTokens.push_back(tok::identifier); 169 ExpectedTokens.push_back(tok::identifier); 170 ExpectedTokens.push_back(tok::identifier); 171 ExpectedTokens.push_back(tok::identifier); 172 173 std::vector<Token> toks = CheckLex("#define M(x) x\n" 174 "M(c M(i)) M(M(i) c)", 175 ExpectedTokens); 176 177 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2])); 178 } 179 180 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) { 181 std::vector<tok::TokenKind> ExpectedTokens; 182 ExpectedTokens.push_back(tok::identifier); 183 ExpectedTokens.push_back(tok::l_paren); 184 ExpectedTokens.push_back(tok::identifier); 185 ExpectedTokens.push_back(tok::r_paren); 186 187 std::vector<Token> toks = CheckLex("#define M(x) x\n" 188 "#define C(x) M(x##c)\n" 189 "M(f(C(i)))", 190 ExpectedTokens); 191 192 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2])); 193 } 194 195 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) { 196 std::vector<tok::TokenKind> ExpectedTokens; 197 ExpectedTokens.push_back(tok::identifier); 198 ExpectedTokens.push_back(tok::l_paren); 199 ExpectedTokens.push_back(tok::identifier); 200 ExpectedTokens.push_back(tok::r_paren); 201 202 std::vector<Token> toks = CheckLex("#define M(x) x\n" 203 "f(M(M(i)))", 204 ExpectedTokens); 205 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2])); 206 } 207 208 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) { 209 std::vector<tok::TokenKind> ExpectedTokens; 210 ExpectedTokens.push_back(tok::identifier); 211 ExpectedTokens.push_back(tok::l_paren); 212 ExpectedTokens.push_back(tok::identifier); 213 ExpectedTokens.push_back(tok::r_paren); 214 215 std::vector<Token> toks = CheckLex("#define M(x) x\n" 216 "M(f(i))", 217 ExpectedTokens); 218 EXPECT_EQ("i", getSourceText(toks[2], toks[2])); 219 } 220 221 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) { 222 std::vector<tok::TokenKind> ExpectedTokens; 223 ExpectedTokens.push_back(tok::identifier); 224 ExpectedTokens.push_back(tok::l_paren); 225 ExpectedTokens.push_back(tok::identifier); 226 ExpectedTokens.push_back(tok::r_paren); 227 228 std::vector<Token> toks = CheckLex("#define M(x) x\n" 229 "#define C(x) x\n" 230 "f(C(M(i)))", 231 ExpectedTokens); 232 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2])); 233 } 234 235 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) { 236 std::vector<tok::TokenKind> ExpectedTokens; 237 ExpectedTokens.push_back(tok::identifier); 238 ExpectedTokens.push_back(tok::l_paren); 239 ExpectedTokens.push_back(tok::identifier); 240 ExpectedTokens.push_back(tok::identifier); 241 ExpectedTokens.push_back(tok::r_paren); 242 243 std::vector<Token> toks = CheckLex("#define M(x) x\n" 244 "#define C(x) c x\n" 245 "f(C(M(i)))", 246 ExpectedTokens); 247 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 248 } 249 250 TEST_F(LexerTest, GetSourceTextExpandsRecursively) { 251 std::vector<tok::TokenKind> ExpectedTokens; 252 ExpectedTokens.push_back(tok::identifier); 253 ExpectedTokens.push_back(tok::identifier); 254 ExpectedTokens.push_back(tok::l_paren); 255 ExpectedTokens.push_back(tok::identifier); 256 ExpectedTokens.push_back(tok::r_paren); 257 258 std::vector<Token> toks = CheckLex("#define M(x) x\n" 259 "#define C(x) c M(x)\n" 260 "C(f(M(i)))", 261 ExpectedTokens); 262 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 263 } 264 265 TEST_F(LexerTest, LexAPI) { 266 std::vector<tok::TokenKind> ExpectedTokens; 267 ExpectedTokens.push_back(tok::l_square); 268 ExpectedTokens.push_back(tok::identifier); 269 ExpectedTokens.push_back(tok::r_square); 270 ExpectedTokens.push_back(tok::l_square); 271 ExpectedTokens.push_back(tok::identifier); 272 ExpectedTokens.push_back(tok::r_square); 273 ExpectedTokens.push_back(tok::identifier); 274 ExpectedTokens.push_back(tok::identifier); 275 ExpectedTokens.push_back(tok::identifier); 276 ExpectedTokens.push_back(tok::identifier); 277 278 std::vector<Token> toks = CheckLex("#define M(x) [x]\n" 279 "#define N(x) x\n" 280 "#define INN(x) x\n" 281 "#define NOF1 INN(val)\n" 282 "#define NOF2 val\n" 283 "M(foo) N([bar])\n" 284 "N(INN(val)) N(NOF1) N(NOF2) N(val)", 285 ExpectedTokens); 286 287 SourceLocation lsqrLoc = toks[0].getLocation(); 288 SourceLocation idLoc = toks[1].getLocation(); 289 SourceLocation rsqrLoc = toks[2].getLocation(); 290 CharSourceRange macroRange = SourceMgr.getExpansionRange(lsqrLoc); 291 292 SourceLocation Loc; 293 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc)); 294 EXPECT_EQ(Loc, macroRange.getBegin()); 295 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 296 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 297 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc)); 298 EXPECT_EQ(Loc, macroRange.getEnd()); 299 EXPECT_TRUE(macroRange.isTokenRange()); 300 301 CharSourceRange range = Lexer::makeFileCharRange( 302 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts); 303 EXPECT_TRUE(range.isInvalid()); 304 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc), 305 SourceMgr, LangOpts); 306 EXPECT_TRUE(range.isInvalid()); 307 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 308 SourceMgr, LangOpts); 309 EXPECT_TRUE(!range.isTokenRange()); 310 EXPECT_EQ(range.getAsRange(), 311 SourceRange(macroRange.getBegin(), 312 macroRange.getEnd().getLocWithOffset(1))); 313 314 StringRef text = Lexer::getSourceText( 315 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 316 SourceMgr, LangOpts); 317 EXPECT_EQ(text, "M(foo)"); 318 319 SourceLocation macroLsqrLoc = toks[3].getLocation(); 320 SourceLocation macroIdLoc = toks[4].getLocation(); 321 SourceLocation macroRsqrLoc = toks[5].getLocation(); 322 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc); 323 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc); 324 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc); 325 326 range = Lexer::makeFileCharRange( 327 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc), 328 SourceMgr, LangOpts); 329 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)), 330 range.getAsRange()); 331 332 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc), 333 SourceMgr, LangOpts); 334 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)), 335 range.getAsRange()); 336 337 macroRange = SourceMgr.getExpansionRange(macroLsqrLoc); 338 range = Lexer::makeFileCharRange( 339 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), 340 SourceMgr, LangOpts); 341 EXPECT_EQ(SourceRange(macroRange.getBegin(), macroRange.getEnd().getLocWithOffset(1)), 342 range.getAsRange()); 343 344 text = Lexer::getSourceText( 345 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)), 346 SourceMgr, LangOpts); 347 EXPECT_EQ(text, "[bar"); 348 349 350 SourceLocation idLoc1 = toks[6].getLocation(); 351 SourceLocation idLoc2 = toks[7].getLocation(); 352 SourceLocation idLoc3 = toks[8].getLocation(); 353 SourceLocation idLoc4 = toks[9].getLocation(); 354 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts)); 355 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts)); 356 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts)); 357 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts)); 358 } 359 360 TEST_F(LexerTest, DontMergeMacroArgsFromDifferentMacroFiles) { 361 std::vector<Token> toks = 362 Lex("#define helper1 0\n" 363 "void helper2(const char *, ...);\n" 364 "#define M1(a, ...) helper2(a, ##__VA_ARGS__)\n" 365 "#define M2(a, ...) M1(a, helper1, ##__VA_ARGS__)\n" 366 "void f1() { M2(\"a\", \"b\"); }"); 367 368 // Check the file corresponding to the "helper1" macro arg in M2. 369 // 370 // The lexer used to report its size as 31, meaning that the end of the 371 // expansion would be on the *next line* (just past `M2("a", "b")`). Make 372 // sure that we get the correct end location (the comma after "helper1"). 373 SourceLocation helper1ArgLoc = toks[20].getLocation(); 374 EXPECT_EQ(SourceMgr.getFileIDSize(SourceMgr.getFileID(helper1ArgLoc)), 8U); 375 } 376 377 TEST_F(LexerTest, DontOverallocateStringifyArgs) { 378 TrivialModuleLoader ModLoader; 379 auto PP = CreatePP("\"StrArg\", 5, 'C'", ModLoader); 380 381 llvm::BumpPtrAllocator Allocator; 382 std::array<IdentifierInfo *, 3> ParamList; 383 MacroInfo *MI = PP->AllocateMacroInfo({}); 384 MI->setIsFunctionLike(); 385 MI->setParameterList(ParamList, Allocator); 386 EXPECT_EQ(3u, MI->getNumParams()); 387 EXPECT_TRUE(MI->isFunctionLike()); 388 389 Token Eof; 390 Eof.setKind(tok::eof); 391 std::vector<Token> ArgTokens; 392 while (1) { 393 Token tok; 394 PP->Lex(tok); 395 if (tok.is(tok::eof)) { 396 ArgTokens.push_back(Eof); 397 break; 398 } 399 if (tok.is(tok::comma)) 400 ArgTokens.push_back(Eof); 401 else 402 ArgTokens.push_back(tok); 403 } 404 405 auto MacroArgsDeleter = [&PP](MacroArgs *M) { M->destroy(*PP); }; 406 std::unique_ptr<MacroArgs, decltype(MacroArgsDeleter)> MA( 407 MacroArgs::create(MI, ArgTokens, false, *PP), MacroArgsDeleter); 408 auto StringifyArg = [&](int ArgNo) { 409 return MA->StringifyArgument(MA->getUnexpArgument(ArgNo), *PP, 410 /*Charify=*/false, {}, {}); 411 }; 412 Token Result = StringifyArg(0); 413 EXPECT_EQ(tok::string_literal, Result.getKind()); 414 EXPECT_STREQ("\"\\\"StrArg\\\"\"", Result.getLiteralData()); 415 Result = StringifyArg(1); 416 EXPECT_EQ(tok::string_literal, Result.getKind()); 417 EXPECT_STREQ("\"5\"", Result.getLiteralData()); 418 Result = StringifyArg(2); 419 EXPECT_EQ(tok::string_literal, Result.getKind()); 420 EXPECT_STREQ("\"'C'\"", Result.getLiteralData()); 421 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST 422 EXPECT_DEATH(StringifyArg(3), "Invalid arg #"); 423 #endif 424 } 425 426 TEST_F(LexerTest, IsNewLineEscapedValid) { 427 auto hasNewLineEscaped = [](const char *S) { 428 return Lexer::isNewLineEscaped(S, S + strlen(S) - 1); 429 }; 430 431 EXPECT_TRUE(hasNewLineEscaped("\\\r")); 432 EXPECT_TRUE(hasNewLineEscaped("\\\n")); 433 EXPECT_TRUE(hasNewLineEscaped("\\\r\n")); 434 EXPECT_TRUE(hasNewLineEscaped("\\\n\r")); 435 EXPECT_TRUE(hasNewLineEscaped("\\ \t\v\f\r")); 436 EXPECT_TRUE(hasNewLineEscaped("\\ \t\v\f\r\n")); 437 438 EXPECT_FALSE(hasNewLineEscaped("\\\r\r")); 439 EXPECT_FALSE(hasNewLineEscaped("\\\r\r\n")); 440 EXPECT_FALSE(hasNewLineEscaped("\\\n\n")); 441 EXPECT_FALSE(hasNewLineEscaped("\r")); 442 EXPECT_FALSE(hasNewLineEscaped("\n")); 443 EXPECT_FALSE(hasNewLineEscaped("\r\n")); 444 EXPECT_FALSE(hasNewLineEscaped("\n\r")); 445 EXPECT_FALSE(hasNewLineEscaped("\r\r")); 446 EXPECT_FALSE(hasNewLineEscaped("\n\n")); 447 } 448 449 TEST_F(LexerTest, GetBeginningOfTokenWithEscapedNewLine) { 450 // Each line should have the same length for 451 // further offset calculation to be more straightforward. 452 const unsigned IdentifierLength = 8; 453 std::string TextToLex = "rabarbar\n" 454 "foo\\\nbar\n" 455 "foo\\\rbar\n" 456 "fo\\\r\nbar\n" 457 "foo\\\n\rba\n"; 458 std::vector<tok::TokenKind> ExpectedTokens{5, tok::identifier}; 459 std::vector<Token> LexedTokens = CheckLex(TextToLex, ExpectedTokens); 460 461 for (const Token &Tok : LexedTokens) { 462 std::pair<FileID, unsigned> OriginalLocation = 463 SourceMgr.getDecomposedLoc(Tok.getLocation()); 464 for (unsigned Offset = 0; Offset < IdentifierLength; ++Offset) { 465 SourceLocation LookupLocation = 466 Tok.getLocation().getLocWithOffset(Offset); 467 468 std::pair<FileID, unsigned> FoundLocation = 469 SourceMgr.getDecomposedExpansionLoc( 470 Lexer::GetBeginningOfToken(LookupLocation, SourceMgr, LangOpts)); 471 472 // Check that location returned by the GetBeginningOfToken 473 // is the same as original token location reported by Lexer. 474 EXPECT_EQ(FoundLocation.second, OriginalLocation.second); 475 } 476 } 477 } 478 479 TEST_F(LexerTest, AvoidPastEndOfStringDereference) { 480 EXPECT_TRUE(Lex(" // \\\n").empty()); 481 EXPECT_TRUE(Lex("#include <\\\\").empty()); 482 EXPECT_TRUE(Lex("#include <\\\\\n").empty()); 483 } 484 485 TEST_F(LexerTest, StringizingRasString) { 486 // For "std::string Lexer::Stringify(StringRef Str, bool Charify)". 487 std::string String1 = R"(foo 488 {"bar":[]} 489 baz)"; 490 // For "void Lexer::Stringify(SmallVectorImpl<char> &Str)". 491 SmallString<128> String2; 492 String2 += String1.c_str(); 493 494 // Corner cases. 495 std::string String3 = R"(\ 496 \n 497 \\n 498 \\)"; 499 SmallString<128> String4; 500 String4 += String3.c_str(); 501 std::string String5 = R"(a\ 502 503 504 \\b)"; 505 SmallString<128> String6; 506 String6 += String5.c_str(); 507 508 String1 = Lexer::Stringify(StringRef(String1)); 509 Lexer::Stringify(String2); 510 String3 = Lexer::Stringify(StringRef(String3)); 511 Lexer::Stringify(String4); 512 String5 = Lexer::Stringify(StringRef(String5)); 513 Lexer::Stringify(String6); 514 515 EXPECT_EQ(String1, R"(foo\n {\"bar\":[]}\n baz)"); 516 EXPECT_EQ(String2, R"(foo\n {\"bar\":[]}\n baz)"); 517 EXPECT_EQ(String3, R"(\\\n \\n\n \\\\n\n \\\\)"); 518 EXPECT_EQ(String4, R"(\\\n \\n\n \\\\n\n \\\\)"); 519 EXPECT_EQ(String5, R"(a\\\n\n\n \\\\b)"); 520 EXPECT_EQ(String6, R"(a\\\n\n\n \\\\b)"); 521 } 522 523 TEST_F(LexerTest, CharRangeOffByOne) { 524 std::vector<Token> toks = Lex(R"(#define MOO 1 525 void foo() { MOO; })"); 526 const Token &moo = toks[5]; 527 528 EXPECT_EQ(getSourceText(moo, moo), "MOO"); 529 530 SourceRange R{moo.getLocation(), moo.getLocation()}; 531 532 EXPECT_TRUE( 533 Lexer::isAtStartOfMacroExpansion(R.getBegin(), SourceMgr, LangOpts)); 534 EXPECT_TRUE( 535 Lexer::isAtEndOfMacroExpansion(R.getEnd(), SourceMgr, LangOpts)); 536 537 CharSourceRange CR = Lexer::getAsCharRange(R, SourceMgr, LangOpts); 538 539 EXPECT_EQ(Lexer::getSourceText(CR, SourceMgr, LangOpts), "MOO"); // Was "MO". 540 } 541 542 TEST_F(LexerTest, FindNextToken) { 543 Lex("int abcd = 0;\n" 544 "int xyz = abcd;\n"); 545 std::vector<std::string> GeneratedByNextToken; 546 SourceLocation Loc = 547 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()); 548 while (true) { 549 auto T = Lexer::findNextToken(Loc, SourceMgr, LangOpts); 550 ASSERT_TRUE(T.hasValue()); 551 if (T->is(tok::eof)) 552 break; 553 GeneratedByNextToken.push_back(getSourceText(*T, *T)); 554 Loc = T->getLocation(); 555 } 556 EXPECT_THAT(GeneratedByNextToken, ElementsAre("abcd", "=", "0", ";", "int", 557 "xyz", "=", "abcd", ";")); 558 } 559 } // anonymous namespace 560