1 //===- unittests/Basic/SourceManagerTest.cpp ------ SourceManager tests ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Basic/SourceManager.h" 10 #include "clang/Basic/Diagnostic.h" 11 #include "clang/Basic/DiagnosticOptions.h" 12 #include "clang/Basic/FileManager.h" 13 #include "clang/Basic/LangOptions.h" 14 #include "clang/Basic/TargetInfo.h" 15 #include "clang/Basic/TargetOptions.h" 16 #include "clang/Lex/HeaderSearch.h" 17 #include "clang/Lex/HeaderSearchOptions.h" 18 #include "clang/Lex/ModuleLoader.h" 19 #include "clang/Lex/Preprocessor.h" 20 #include "clang/Lex/PreprocessorOptions.h" 21 #include "llvm/ADT/SmallString.h" 22 #include "llvm/Config/llvm-config.h" 23 #include "llvm/Support/Process.h" 24 #include "gtest/gtest.h" 25 #include <cstddef> 26 27 using namespace clang; 28 29 namespace { 30 31 // The test fixture. 32 class SourceManagerTest : public ::testing::Test { 33 protected: 34 SourceManagerTest() 35 : FileMgr(FileMgrOpts), 36 DiagID(new DiagnosticIDs()), 37 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), 38 SourceMgr(Diags, FileMgr), 39 TargetOpts(new TargetOptions) { 40 TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; 41 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); 42 } 43 44 FileSystemOptions FileMgrOpts; 45 FileManager FileMgr; 46 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 47 DiagnosticsEngine Diags; 48 SourceManager SourceMgr; 49 LangOptions LangOpts; 50 std::shared_ptr<TargetOptions> TargetOpts; 51 IntrusiveRefCntPtr<TargetInfo> Target; 52 }; 53 54 TEST_F(SourceManagerTest, isBeforeInTranslationUnit) { 55 const char *source = 56 "#define M(x) [x]\n" 57 "M(foo)"; 58 std::unique_ptr<llvm::MemoryBuffer> Buf = 59 llvm::MemoryBuffer::getMemBuffer(source); 60 FileID mainFileID = SourceMgr.createFileID(std::move(Buf)); 61 SourceMgr.setMainFileID(mainFileID); 62 63 TrivialModuleLoader ModLoader; 64 HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, 65 Diags, LangOpts, &*Target); 66 Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts, 67 SourceMgr, HeaderInfo, ModLoader, 68 /*IILookup =*/nullptr, 69 /*OwnsHeaderSearch =*/false); 70 PP.Initialize(*Target); 71 PP.EnterMainSourceFile(); 72 73 std::vector<Token> toks; 74 while (1) { 75 Token tok; 76 PP.Lex(tok); 77 if (tok.is(tok::eof)) 78 break; 79 toks.push_back(tok); 80 } 81 82 // Make sure we got the tokens that we expected. 83 ASSERT_EQ(3U, toks.size()); 84 ASSERT_EQ(tok::l_square, toks[0].getKind()); 85 ASSERT_EQ(tok::identifier, toks[1].getKind()); 86 ASSERT_EQ(tok::r_square, toks[2].getKind()); 87 88 SourceLocation lsqrLoc = toks[0].getLocation(); 89 SourceLocation idLoc = toks[1].getLocation(); 90 SourceLocation rsqrLoc = toks[2].getLocation(); 91 92 SourceLocation macroExpStartLoc = SourceMgr.translateLineCol(mainFileID, 2, 1); 93 SourceLocation macroExpEndLoc = SourceMgr.translateLineCol(mainFileID, 2, 6); 94 ASSERT_TRUE(macroExpStartLoc.isFileID()); 95 ASSERT_TRUE(macroExpEndLoc.isFileID()); 96 97 SmallString<32> str; 98 ASSERT_EQ("M", PP.getSpelling(macroExpStartLoc, str)); 99 ASSERT_EQ(")", PP.getSpelling(macroExpEndLoc, str)); 100 101 EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(lsqrLoc, idLoc)); 102 EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(idLoc, rsqrLoc)); 103 EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(macroExpStartLoc, idLoc)); 104 EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(idLoc, macroExpEndLoc)); 105 } 106 107 TEST_F(SourceManagerTest, getColumnNumber) { 108 const char *Source = 109 "int x;\n" 110 "int y;"; 111 112 std::unique_ptr<llvm::MemoryBuffer> Buf = 113 llvm::MemoryBuffer::getMemBuffer(Source); 114 FileID MainFileID = SourceMgr.createFileID(std::move(Buf)); 115 SourceMgr.setMainFileID(MainFileID); 116 117 bool Invalid; 118 119 Invalid = false; 120 EXPECT_EQ(1U, SourceMgr.getColumnNumber(MainFileID, 0, &Invalid)); 121 EXPECT_TRUE(!Invalid); 122 123 Invalid = false; 124 EXPECT_EQ(5U, SourceMgr.getColumnNumber(MainFileID, 4, &Invalid)); 125 EXPECT_TRUE(!Invalid); 126 127 Invalid = false; 128 EXPECT_EQ(1U, SourceMgr.getColumnNumber(MainFileID, 7, &Invalid)); 129 EXPECT_TRUE(!Invalid); 130 131 Invalid = false; 132 EXPECT_EQ(5U, SourceMgr.getColumnNumber(MainFileID, 11, &Invalid)); 133 EXPECT_TRUE(!Invalid); 134 135 Invalid = false; 136 EXPECT_EQ(7U, SourceMgr.getColumnNumber(MainFileID, strlen(Source), 137 &Invalid)); 138 EXPECT_TRUE(!Invalid); 139 140 Invalid = false; 141 SourceMgr.getColumnNumber(MainFileID, strlen(Source)+1, &Invalid); 142 EXPECT_TRUE(Invalid); 143 144 // Test invalid files 145 Invalid = false; 146 SourceMgr.getColumnNumber(FileID(), 0, &Invalid); 147 EXPECT_TRUE(Invalid); 148 149 Invalid = false; 150 SourceMgr.getColumnNumber(FileID(), 1, &Invalid); 151 EXPECT_TRUE(Invalid); 152 153 // Test with no invalid flag. 154 EXPECT_EQ(1U, SourceMgr.getColumnNumber(MainFileID, 0, nullptr)); 155 } 156 157 TEST_F(SourceManagerTest, locationPrintTest) { 158 const char *header = "#define IDENTITY(x) x\n"; 159 160 const char *Source = "int x;\n" 161 "include \"test-header.h\"\n" 162 "IDENTITY(int y);\n" 163 "int z;"; 164 165 std::unique_ptr<llvm::MemoryBuffer> HeaderBuf = 166 llvm::MemoryBuffer::getMemBuffer(header); 167 std::unique_ptr<llvm::MemoryBuffer> Buf = 168 llvm::MemoryBuffer::getMemBuffer(Source); 169 170 const FileEntry *SourceFile = 171 FileMgr.getVirtualFile("/mainFile.cpp", Buf->getBufferSize(), 0); 172 SourceMgr.overrideFileContents(SourceFile, std::move(Buf)); 173 174 const FileEntry *HeaderFile = 175 FileMgr.getVirtualFile("/test-header.h", HeaderBuf->getBufferSize(), 0); 176 SourceMgr.overrideFileContents(HeaderFile, std::move(HeaderBuf)); 177 178 FileID MainFileID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User); 179 FileID HeaderFileID = SourceMgr.getOrCreateFileID(HeaderFile, SrcMgr::C_User); 180 SourceMgr.setMainFileID(MainFileID); 181 182 auto BeginLoc = SourceMgr.getLocForStartOfFile(MainFileID); 183 auto EndLoc = SourceMgr.getLocForEndOfFile(MainFileID); 184 185 auto BeginEOLLoc = SourceMgr.translateLineCol(MainFileID, 1, 7); 186 187 auto HeaderLoc = SourceMgr.getLocForStartOfFile(HeaderFileID); 188 189 EXPECT_EQ(BeginLoc.printToString(SourceMgr), "/mainFile.cpp:1:1"); 190 EXPECT_EQ(EndLoc.printToString(SourceMgr), "/mainFile.cpp:4:7"); 191 192 EXPECT_EQ(BeginEOLLoc.printToString(SourceMgr), "/mainFile.cpp:1:7"); 193 EXPECT_EQ(HeaderLoc.printToString(SourceMgr), "/test-header.h:1:1"); 194 195 EXPECT_EQ(SourceRange(BeginLoc, BeginLoc).printToString(SourceMgr), 196 "</mainFile.cpp:1:1>"); 197 EXPECT_EQ(SourceRange(BeginLoc, BeginEOLLoc).printToString(SourceMgr), 198 "</mainFile.cpp:1:1, col:7>"); 199 EXPECT_EQ(SourceRange(BeginLoc, EndLoc).printToString(SourceMgr), 200 "</mainFile.cpp:1:1, line:4:7>"); 201 EXPECT_EQ(SourceRange(BeginLoc, HeaderLoc).printToString(SourceMgr), 202 "</mainFile.cpp:1:1, /test-header.h:1:1>"); 203 } 204 205 TEST_F(SourceManagerTest, getInvalidBOM) { 206 ASSERT_EQ(SrcMgr::ContentCache::getInvalidBOM(""), nullptr); 207 ASSERT_EQ(SrcMgr::ContentCache::getInvalidBOM("\x00\x00\x00"), nullptr); 208 ASSERT_EQ(SrcMgr::ContentCache::getInvalidBOM("\xFF\xFF\xFF"), nullptr); 209 ASSERT_EQ(SrcMgr::ContentCache::getInvalidBOM("#include <iostream>"), 210 nullptr); 211 212 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 213 "\xFE\xFF#include <iostream>")), 214 "UTF-16 (BE)"); 215 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 216 "\xFF\xFE#include <iostream>")), 217 "UTF-16 (LE)"); 218 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 219 "\x2B\x2F\x76#include <iostream>")), 220 "UTF-7"); 221 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 222 "\xF7\x64\x4C#include <iostream>")), 223 "UTF-1"); 224 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 225 "\xDD\x73\x66\x73#include <iostream>")), 226 "UTF-EBCDIC"); 227 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 228 "\x0E\xFE\xFF#include <iostream>")), 229 "SCSU"); 230 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 231 "\xFB\xEE\x28#include <iostream>")), 232 "BOCU-1"); 233 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 234 "\x84\x31\x95\x33#include <iostream>")), 235 "GB-18030"); 236 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 237 llvm::StringLiteral::withInnerNUL( 238 "\x00\x00\xFE\xFF#include <iostream>"))), 239 "UTF-32 (BE)"); 240 ASSERT_EQ(StringRef(SrcMgr::ContentCache::getInvalidBOM( 241 llvm::StringLiteral::withInnerNUL( 242 "\xFF\xFE\x00\x00#include <iostream>"))), 243 "UTF-32 (LE)"); 244 } 245 246 // Regression test - there was an out of bound access for buffers not terminated by zero. 247 TEST_F(SourceManagerTest, getLineNumber) { 248 const unsigned pageSize = llvm::sys::Process::getPageSizeEstimate(); 249 std::unique_ptr<char[]> source(new char[pageSize]); 250 for(unsigned i = 0; i < pageSize; ++i) { 251 source[i] = 'a'; 252 } 253 254 std::unique_ptr<llvm::MemoryBuffer> Buf = 255 llvm::MemoryBuffer::getMemBuffer( 256 llvm::MemoryBufferRef( 257 llvm::StringRef(source.get(), 3), "whatever" 258 ), 259 false 260 ); 261 262 FileID mainFileID = SourceMgr.createFileID(std::move(Buf)); 263 SourceMgr.setMainFileID(mainFileID); 264 265 ASSERT_NO_FATAL_FAILURE(SourceMgr.getLineNumber(mainFileID, 1, nullptr)); 266 } 267 268 #if defined(LLVM_ON_UNIX) 269 270 TEST_F(SourceManagerTest, getMacroArgExpandedLocation) { 271 const char *header = 272 "#define FM(x,y) x\n"; 273 274 const char *main = 275 "#include \"/test-header.h\"\n" 276 "#define VAL 0\n" 277 "FM(VAL,0)\n" 278 "FM(0,VAL)\n" 279 "FM(FM(0,VAL),0)\n" 280 "#define CONCAT(X, Y) X##Y\n" 281 "CONCAT(1,1)\n"; 282 283 std::unique_ptr<llvm::MemoryBuffer> HeaderBuf = 284 llvm::MemoryBuffer::getMemBuffer(header); 285 std::unique_ptr<llvm::MemoryBuffer> MainBuf = 286 llvm::MemoryBuffer::getMemBuffer(main); 287 FileID mainFileID = SourceMgr.createFileID(std::move(MainBuf)); 288 SourceMgr.setMainFileID(mainFileID); 289 290 const FileEntry *headerFile = FileMgr.getVirtualFile("/test-header.h", 291 HeaderBuf->getBufferSize(), 0); 292 SourceMgr.overrideFileContents(headerFile, std::move(HeaderBuf)); 293 294 TrivialModuleLoader ModLoader; 295 HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, 296 Diags, LangOpts, &*Target); 297 Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts, 298 SourceMgr, HeaderInfo, ModLoader, 299 /*IILookup =*/nullptr, 300 /*OwnsHeaderSearch =*/false); 301 PP.Initialize(*Target); 302 PP.EnterMainSourceFile(); 303 304 std::vector<Token> toks; 305 while (1) { 306 Token tok; 307 PP.Lex(tok); 308 if (tok.is(tok::eof)) 309 break; 310 toks.push_back(tok); 311 } 312 313 // Make sure we got the tokens that we expected. 314 ASSERT_EQ(4U, toks.size()); 315 ASSERT_EQ(tok::numeric_constant, toks[0].getKind()); 316 ASSERT_EQ(tok::numeric_constant, toks[1].getKind()); 317 ASSERT_EQ(tok::numeric_constant, toks[2].getKind()); 318 ASSERT_EQ(tok::numeric_constant, toks[3].getKind()); 319 320 SourceLocation defLoc = SourceMgr.translateLineCol(mainFileID, 2, 13); 321 SourceLocation loc1 = SourceMgr.translateLineCol(mainFileID, 3, 8); 322 SourceLocation loc2 = SourceMgr.translateLineCol(mainFileID, 4, 4); 323 SourceLocation loc3 = SourceMgr.translateLineCol(mainFileID, 5, 7); 324 SourceLocation defLoc2 = SourceMgr.translateLineCol(mainFileID, 6, 22); 325 defLoc = SourceMgr.getMacroArgExpandedLocation(defLoc); 326 loc1 = SourceMgr.getMacroArgExpandedLocation(loc1); 327 loc2 = SourceMgr.getMacroArgExpandedLocation(loc2); 328 loc3 = SourceMgr.getMacroArgExpandedLocation(loc3); 329 defLoc2 = SourceMgr.getMacroArgExpandedLocation(defLoc2); 330 331 EXPECT_TRUE(defLoc.isFileID()); 332 EXPECT_TRUE(loc1.isFileID()); 333 EXPECT_TRUE(SourceMgr.isMacroArgExpansion(loc2)); 334 EXPECT_TRUE(SourceMgr.isMacroArgExpansion(loc3)); 335 EXPECT_EQ(loc2, toks[1].getLocation()); 336 EXPECT_EQ(loc3, toks[2].getLocation()); 337 EXPECT_TRUE(defLoc2.isFileID()); 338 } 339 340 namespace { 341 342 struct MacroAction { 343 enum Kind { kExpansion, kDefinition, kUnDefinition}; 344 345 SourceLocation Loc; 346 std::string Name; 347 unsigned MAKind : 3; 348 349 MacroAction(SourceLocation Loc, StringRef Name, unsigned K) 350 : Loc(Loc), Name(Name), MAKind(K) { } 351 352 bool isExpansion() const { return MAKind == kExpansion; } 353 bool isDefinition() const { return MAKind & kDefinition; } 354 bool isUnDefinition() const { return MAKind & kUnDefinition; } 355 }; 356 357 class MacroTracker : public PPCallbacks { 358 std::vector<MacroAction> &Macros; 359 360 public: 361 explicit MacroTracker(std::vector<MacroAction> &Macros) : Macros(Macros) { } 362 363 void MacroDefined(const Token &MacroNameTok, 364 const MacroDirective *MD) override { 365 Macros.push_back(MacroAction(MD->getLocation(), 366 MacroNameTok.getIdentifierInfo()->getName(), 367 MacroAction::kDefinition)); 368 } 369 void MacroUndefined(const Token &MacroNameTok, 370 const MacroDefinition &MD, 371 const MacroDirective *UD) override { 372 Macros.push_back( 373 MacroAction(UD ? UD->getLocation() : SourceLocation(), 374 MacroNameTok.getIdentifierInfo()->getName(), 375 UD ? MacroAction::kDefinition | MacroAction::kUnDefinition 376 : MacroAction::kUnDefinition)); 377 } 378 void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD, 379 SourceRange Range, const MacroArgs *Args) override { 380 Macros.push_back(MacroAction(MacroNameTok.getLocation(), 381 MacroNameTok.getIdentifierInfo()->getName(), 382 MacroAction::kExpansion)); 383 } 384 }; 385 386 } 387 388 TEST_F(SourceManagerTest, isBeforeInTranslationUnitWithMacroInInclude) { 389 const char *header = 390 "#define MACRO_IN_INCLUDE 0\n" 391 "#define MACRO_DEFINED\n" 392 "#undef MACRO_DEFINED\n" 393 "#undef MACRO_UNDEFINED\n"; 394 395 const char *main = 396 "#define M(x) x\n" 397 "#define INC \"/test-header.h\"\n" 398 "#include M(INC)\n" 399 "#define INC2 </test-header.h>\n" 400 "#include M(INC2)\n"; 401 402 std::unique_ptr<llvm::MemoryBuffer> HeaderBuf = 403 llvm::MemoryBuffer::getMemBuffer(header); 404 std::unique_ptr<llvm::MemoryBuffer> MainBuf = 405 llvm::MemoryBuffer::getMemBuffer(main); 406 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(MainBuf))); 407 408 const FileEntry *headerFile = FileMgr.getVirtualFile("/test-header.h", 409 HeaderBuf->getBufferSize(), 0); 410 SourceMgr.overrideFileContents(headerFile, std::move(HeaderBuf)); 411 412 TrivialModuleLoader ModLoader; 413 HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, 414 Diags, LangOpts, &*Target); 415 Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts, 416 SourceMgr, HeaderInfo, ModLoader, 417 /*IILookup =*/nullptr, 418 /*OwnsHeaderSearch =*/false); 419 PP.Initialize(*Target); 420 421 std::vector<MacroAction> Macros; 422 PP.addPPCallbacks(std::make_unique<MacroTracker>(Macros)); 423 424 PP.EnterMainSourceFile(); 425 426 std::vector<Token> toks; 427 while (1) { 428 Token tok; 429 PP.Lex(tok); 430 if (tok.is(tok::eof)) 431 break; 432 toks.push_back(tok); 433 } 434 435 // Make sure we got the tokens that we expected. 436 ASSERT_EQ(0U, toks.size()); 437 438 ASSERT_EQ(15U, Macros.size()); 439 // #define M(x) x 440 ASSERT_TRUE(Macros[0].isDefinition()); 441 ASSERT_EQ("M", Macros[0].Name); 442 // #define INC "/test-header.h" 443 ASSERT_TRUE(Macros[1].isDefinition()); 444 ASSERT_EQ("INC", Macros[1].Name); 445 // M expansion in #include M(INC) 446 ASSERT_FALSE(Macros[2].isDefinition()); 447 ASSERT_EQ("M", Macros[2].Name); 448 // INC expansion in #include M(INC) 449 ASSERT_TRUE(Macros[3].isExpansion()); 450 ASSERT_EQ("INC", Macros[3].Name); 451 // #define MACRO_IN_INCLUDE 0 452 ASSERT_TRUE(Macros[4].isDefinition()); 453 ASSERT_EQ("MACRO_IN_INCLUDE", Macros[4].Name); 454 // #define MACRO_DEFINED 455 ASSERT_TRUE(Macros[5].isDefinition()); 456 ASSERT_FALSE(Macros[5].isUnDefinition()); 457 ASSERT_EQ("MACRO_DEFINED", Macros[5].Name); 458 // #undef MACRO_DEFINED 459 ASSERT_TRUE(Macros[6].isDefinition()); 460 ASSERT_TRUE(Macros[6].isUnDefinition()); 461 ASSERT_EQ("MACRO_DEFINED", Macros[6].Name); 462 // #undef MACRO_UNDEFINED 463 ASSERT_FALSE(Macros[7].isDefinition()); 464 ASSERT_TRUE(Macros[7].isUnDefinition()); 465 ASSERT_EQ("MACRO_UNDEFINED", Macros[7].Name); 466 // #define INC2 </test-header.h> 467 ASSERT_TRUE(Macros[8].isDefinition()); 468 ASSERT_EQ("INC2", Macros[8].Name); 469 // M expansion in #include M(INC2) 470 ASSERT_FALSE(Macros[9].isDefinition()); 471 ASSERT_EQ("M", Macros[9].Name); 472 // INC2 expansion in #include M(INC2) 473 ASSERT_TRUE(Macros[10].isExpansion()); 474 ASSERT_EQ("INC2", Macros[10].Name); 475 // #define MACRO_IN_INCLUDE 0 476 ASSERT_TRUE(Macros[11].isDefinition()); 477 ASSERT_EQ("MACRO_IN_INCLUDE", Macros[11].Name); 478 479 // The INC expansion in #include M(INC) comes before the first 480 // MACRO_IN_INCLUDE definition of the included file. 481 EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(Macros[3].Loc, Macros[4].Loc)); 482 483 // The INC2 expansion in #include M(INC2) comes before the second 484 // MACRO_IN_INCLUDE definition of the included file. 485 EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(Macros[10].Loc, Macros[11].Loc)); 486 } 487 488 #endif 489 490 } // anonymous namespace 491