1 //===-- SourceCodeTests.cpp ------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 #include "Annotations.h" 9 #include "Protocol.h" 10 #include "SourceCode.h" 11 #include "TestTU.h" 12 #include "support/Context.h" 13 #include "clang/Basic/LangOptions.h" 14 #include "clang/Basic/SourceLocation.h" 15 #include "clang/Basic/TokenKinds.h" 16 #include "clang/Format/Format.h" 17 #include "llvm/Support/Error.h" 18 #include "llvm/Testing/Support/Annotations.h" 19 #include "llvm/Testing/Support/Error.h" 20 #include "gmock/gmock.h" 21 #include "gtest/gtest.h" 22 #include <tuple> 23 24 namespace clang { 25 namespace clangd { 26 namespace { 27 28 using llvm::Failed; 29 using llvm::FailedWithMessage; 30 using llvm::HasValue; 31 32 MATCHER_P2(Pos, Line, Col, "") { 33 return arg.line == int(Line) && arg.character == int(Col); 34 } 35 36 MATCHER_P(macroName, Name, "") { return arg.Name == Name; } 37 38 /// A helper to make tests easier to read. 39 Position position(int Line, int Character) { 40 Position Pos; 41 Pos.line = Line; 42 Pos.character = Character; 43 return Pos; 44 } 45 46 TEST(SourceCodeTests, lspLength) { 47 EXPECT_EQ(lspLength(""), 0UL); 48 EXPECT_EQ(lspLength("ascii"), 5UL); 49 // BMP 50 EXPECT_EQ(lspLength("↓"), 1UL); 51 EXPECT_EQ(lspLength("¥"), 1UL); 52 // astral 53 EXPECT_EQ(lspLength("��"), 2UL); 54 55 WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8); 56 EXPECT_EQ(lspLength(""), 0UL); 57 EXPECT_EQ(lspLength("ascii"), 5UL); 58 // BMP 59 EXPECT_EQ(lspLength("↓"), 3UL); 60 EXPECT_EQ(lspLength("¥"), 2UL); 61 // astral 62 EXPECT_EQ(lspLength("��"), 4UL); 63 64 WithContextValue UTF32(kCurrentOffsetEncoding, OffsetEncoding::UTF32); 65 EXPECT_EQ(lspLength(""), 0UL); 66 EXPECT_EQ(lspLength("ascii"), 5UL); 67 // BMP 68 EXPECT_EQ(lspLength("↓"), 1UL); 69 EXPECT_EQ(lspLength("¥"), 1UL); 70 // astral 71 EXPECT_EQ(lspLength("��"), 1UL); 72 } 73 74 TEST(SourceCodeTests, lspLengthBadUTF8) { 75 // Results are not well-defined if source file isn't valid UTF-8. 76 // However we shouldn't crash or return something totally wild. 77 const char *BadUTF8[] = {"\xa0", "\xff\xff\xff\xff\xff"}; 78 79 for (OffsetEncoding Encoding : 80 {OffsetEncoding::UTF8, OffsetEncoding::UTF16, OffsetEncoding::UTF32}) { 81 WithContextValue UTF32(kCurrentOffsetEncoding, Encoding); 82 for (const char *Bad : BadUTF8) { 83 EXPECT_GE(lspLength(Bad), 0u); 84 EXPECT_LE(lspLength(Bad), strlen(Bad)); 85 } 86 } 87 } 88 89 // The = → �� below are ASCII (1 byte), BMP (3 bytes), and astral (4 bytes). 90 const char File[] = R"(0:0 = 0 91 1:0 → 8 92 2:0 �� 18)"; 93 struct Line { 94 unsigned Number; 95 unsigned Offset; 96 unsigned Length; 97 }; 98 Line FileLines[] = {Line{0, 0, 7}, Line{1, 8, 9}, Line{2, 18, 11}}; 99 100 TEST(SourceCodeTests, PositionToOffset) { 101 // line out of bounds 102 EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed()); 103 // first line 104 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, -1)), 105 llvm::Failed()); // out of range 106 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 0)), 107 llvm::HasValue(0)); // first character 108 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 3)), 109 llvm::HasValue(3)); // middle character 110 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 6)), 111 llvm::HasValue(6)); // last character 112 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7)), 113 llvm::HasValue(7)); // the newline itself 114 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7), false), 115 llvm::HasValue(7)); 116 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8)), 117 llvm::HasValue(7)); // out of range 118 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8), false), 119 llvm::Failed()); // out of range 120 // middle line 121 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, -1)), 122 llvm::Failed()); // out of range 123 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 0)), 124 llvm::HasValue(8)); // first character 125 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3)), 126 llvm::HasValue(11)); // middle character 127 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3), false), 128 llvm::HasValue(11)); 129 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 6)), 130 llvm::HasValue(16)); // last character 131 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 7)), 132 llvm::HasValue(17)); // the newline itself 133 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8)), 134 llvm::HasValue(17)); // out of range 135 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8), false), 136 llvm::Failed()); // out of range 137 // last line 138 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, -1)), 139 llvm::Failed()); // out of range 140 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 0)), 141 llvm::HasValue(18)); // first character 142 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 3)), 143 llvm::HasValue(21)); // middle character 144 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 5), false), 145 llvm::Failed()); // middle of surrogate pair 146 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 5)), 147 llvm::HasValue(26)); // middle of surrogate pair 148 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 6), false), 149 llvm::HasValue(26)); // end of surrogate pair 150 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 8)), 151 llvm::HasValue(28)); // last character 152 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 9)), 153 llvm::HasValue(29)); // EOF 154 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 10), false), 155 llvm::Failed()); // out of range 156 // line out of bounds 157 EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed()); 158 EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 1)), llvm::Failed()); 159 160 // Codepoints are similar, except near astral characters. 161 WithContextValue UTF32(kCurrentOffsetEncoding, OffsetEncoding::UTF32); 162 // line out of bounds 163 EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed()); 164 // first line 165 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, -1)), 166 llvm::Failed()); // out of range 167 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 0)), 168 llvm::HasValue(0)); // first character 169 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 3)), 170 llvm::HasValue(3)); // middle character 171 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 6)), 172 llvm::HasValue(6)); // last character 173 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7)), 174 llvm::HasValue(7)); // the newline itself 175 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7), false), 176 llvm::HasValue(7)); 177 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8)), 178 llvm::HasValue(7)); // out of range 179 EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8), false), 180 llvm::Failed()); // out of range 181 // middle line 182 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, -1)), 183 llvm::Failed()); // out of range 184 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 0)), 185 llvm::HasValue(8)); // first character 186 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3)), 187 llvm::HasValue(11)); // middle character 188 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3), false), 189 llvm::HasValue(11)); 190 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 6)), 191 llvm::HasValue(16)); // last character 192 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 7)), 193 llvm::HasValue(17)); // the newline itself 194 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8)), 195 llvm::HasValue(17)); // out of range 196 EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8), false), 197 llvm::Failed()); // out of range 198 // last line 199 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, -1)), 200 llvm::Failed()); // out of range 201 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 0)), 202 llvm::HasValue(18)); // first character 203 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 4)), 204 llvm::HasValue(22)); // Before astral character. 205 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 5), false), 206 llvm::HasValue(26)); // after astral character 207 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 7)), 208 llvm::HasValue(28)); // last character 209 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 8)), 210 llvm::HasValue(29)); // EOF 211 EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 9), false), 212 llvm::Failed()); // out of range 213 // line out of bounds 214 EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed()); 215 EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 1)), llvm::Failed()); 216 217 // Test UTF-8, where transformations are trivial. 218 WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8); 219 EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed()); 220 EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed()); 221 for (Line L : FileLines) { 222 EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, -1)), 223 llvm::Failed()); // out of range 224 for (unsigned I = 0; I <= L.Length; ++I) 225 EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, I)), 226 llvm::HasValue(L.Offset + I)); 227 EXPECT_THAT_EXPECTED( 228 positionToOffset(File, position(L.Number, L.Length + 1)), 229 llvm::HasValue(L.Offset + L.Length)); 230 EXPECT_THAT_EXPECTED( 231 positionToOffset(File, position(L.Number, L.Length + 1), false), 232 llvm::Failed()); // out of range 233 } 234 } 235 236 TEST(SourceCodeTests, OffsetToPosition) { 237 EXPECT_THAT(offsetToPosition(File, 0), Pos(0, 0)) << "start of file"; 238 EXPECT_THAT(offsetToPosition(File, 3), Pos(0, 3)) << "in first line"; 239 EXPECT_THAT(offsetToPosition(File, 6), Pos(0, 6)) << "end of first line"; 240 EXPECT_THAT(offsetToPosition(File, 7), Pos(0, 7)) << "first newline"; 241 EXPECT_THAT(offsetToPosition(File, 8), Pos(1, 0)) << "start of second line"; 242 EXPECT_THAT(offsetToPosition(File, 12), Pos(1, 4)) << "before BMP char"; 243 EXPECT_THAT(offsetToPosition(File, 13), Pos(1, 5)) << "in BMP char"; 244 EXPECT_THAT(offsetToPosition(File, 15), Pos(1, 5)) << "after BMP char"; 245 EXPECT_THAT(offsetToPosition(File, 16), Pos(1, 6)) << "end of second line"; 246 EXPECT_THAT(offsetToPosition(File, 17), Pos(1, 7)) << "second newline"; 247 EXPECT_THAT(offsetToPosition(File, 18), Pos(2, 0)) << "start of last line"; 248 EXPECT_THAT(offsetToPosition(File, 21), Pos(2, 3)) << "in last line"; 249 EXPECT_THAT(offsetToPosition(File, 22), Pos(2, 4)) << "before astral char"; 250 EXPECT_THAT(offsetToPosition(File, 24), Pos(2, 6)) << "in astral char"; 251 EXPECT_THAT(offsetToPosition(File, 26), Pos(2, 6)) << "after astral char"; 252 EXPECT_THAT(offsetToPosition(File, 28), Pos(2, 8)) << "end of last line"; 253 EXPECT_THAT(offsetToPosition(File, 29), Pos(2, 9)) << "EOF"; 254 EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 9)) << "out of bounds"; 255 256 // Codepoints are similar, except near astral characters. 257 WithContextValue UTF32(kCurrentOffsetEncoding, OffsetEncoding::UTF32); 258 EXPECT_THAT(offsetToPosition(File, 0), Pos(0, 0)) << "start of file"; 259 EXPECT_THAT(offsetToPosition(File, 3), Pos(0, 3)) << "in first line"; 260 EXPECT_THAT(offsetToPosition(File, 6), Pos(0, 6)) << "end of first line"; 261 EXPECT_THAT(offsetToPosition(File, 7), Pos(0, 7)) << "first newline"; 262 EXPECT_THAT(offsetToPosition(File, 8), Pos(1, 0)) << "start of second line"; 263 EXPECT_THAT(offsetToPosition(File, 12), Pos(1, 4)) << "before BMP char"; 264 EXPECT_THAT(offsetToPosition(File, 13), Pos(1, 5)) << "in BMP char"; 265 EXPECT_THAT(offsetToPosition(File, 15), Pos(1, 5)) << "after BMP char"; 266 EXPECT_THAT(offsetToPosition(File, 16), Pos(1, 6)) << "end of second line"; 267 EXPECT_THAT(offsetToPosition(File, 17), Pos(1, 7)) << "second newline"; 268 EXPECT_THAT(offsetToPosition(File, 18), Pos(2, 0)) << "start of last line"; 269 EXPECT_THAT(offsetToPosition(File, 21), Pos(2, 3)) << "in last line"; 270 EXPECT_THAT(offsetToPosition(File, 22), Pos(2, 4)) << "before astral char"; 271 EXPECT_THAT(offsetToPosition(File, 24), Pos(2, 5)) << "in astral char"; 272 EXPECT_THAT(offsetToPosition(File, 26), Pos(2, 5)) << "after astral char"; 273 EXPECT_THAT(offsetToPosition(File, 28), Pos(2, 7)) << "end of last line"; 274 EXPECT_THAT(offsetToPosition(File, 29), Pos(2, 8)) << "EOF"; 275 EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 8)) << "out of bounds"; 276 277 WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8); 278 for (Line L : FileLines) { 279 for (unsigned I = 0; I <= L.Length; ++I) 280 EXPECT_THAT(offsetToPosition(File, L.Offset + I), Pos(L.Number, I)); 281 } 282 EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 11)) << "out of bounds"; 283 } 284 285 TEST(SourceCodeTests, SourceLocationInMainFile) { 286 Annotations Source(R"cpp( 287 ^in^t ^foo 288 ^bar 289 ^baz ^() {} {} {} {} { }^ 290 )cpp"); 291 292 SourceManagerForFile Owner("foo.cpp", Source.code()); 293 SourceManager &SM = Owner.get(); 294 295 SourceLocation StartOfFile = SM.getLocForStartOfFile(SM.getMainFileID()); 296 EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(0, 0)), 297 HasValue(StartOfFile)); 298 // End of file. 299 EXPECT_THAT_EXPECTED( 300 sourceLocationInMainFile(SM, position(4, 0)), 301 HasValue(StartOfFile.getLocWithOffset(Source.code().size()))); 302 // Column number is too large. 303 EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(0, 1)), Failed()); 304 EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(0, 100)), 305 Failed()); 306 EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(4, 1)), Failed()); 307 // Line number is too large. 308 EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(5, 0)), Failed()); 309 // Check all positions mentioned in the test return valid results. 310 for (auto P : Source.points()) { 311 size_t Offset = llvm::cantFail(positionToOffset(Source.code(), P)); 312 EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, P), 313 HasValue(StartOfFile.getLocWithOffset(Offset))); 314 } 315 } 316 317 TEST(SourceCodeTests, isReservedName) { 318 EXPECT_FALSE(isReservedName("")); 319 EXPECT_FALSE(isReservedName("_")); 320 EXPECT_FALSE(isReservedName("foo")); 321 EXPECT_FALSE(isReservedName("_foo")); 322 EXPECT_TRUE(isReservedName("__foo")); 323 EXPECT_TRUE(isReservedName("_Foo")); 324 EXPECT_FALSE(isReservedName("foo__bar")) << "FIXME"; 325 } 326 327 TEST(SourceCodeTests, CollectIdentifiers) { 328 auto Style = format::getLLVMStyle(); 329 auto IDs = collectIdentifiers(R"cpp( 330 #include "a.h" 331 void foo() { int xyz; int abc = xyz; return foo(); } 332 )cpp", 333 Style); 334 EXPECT_EQ(IDs.size(), 7u); 335 EXPECT_EQ(IDs["include"], 1u); 336 EXPECT_EQ(IDs["void"], 1u); 337 EXPECT_EQ(IDs["int"], 2u); 338 EXPECT_EQ(IDs["xyz"], 2u); 339 EXPECT_EQ(IDs["abc"], 1u); 340 EXPECT_EQ(IDs["return"], 1u); 341 EXPECT_EQ(IDs["foo"], 2u); 342 } 343 344 TEST(SourceCodeTests, CollectWords) { 345 auto Words = collectWords(R"cpp( 346 #define FIZZ_BUZZ 347 // this is a comment 348 std::string getSomeText() { return "magic word"; } 349 )cpp"); 350 std::set<StringRef> ActualWords(Words.keys().begin(), Words.keys().end()); 351 std::set<StringRef> ExpectedWords = {"define", "fizz", "buzz", "this", 352 "comment", "string", "some", "text", 353 "return", "magic", "word"}; 354 EXPECT_EQ(ActualWords, ExpectedWords); 355 } 356 357 class SpelledWordsTest : public ::testing::Test { 358 llvm::Optional<ParsedAST> AST; 359 360 llvm::Optional<SpelledWord> tryWord(const char *Text) { 361 llvm::Annotations A(Text); 362 auto TU = TestTU::withCode(A.code()); 363 AST = TU.build(); 364 auto SW = SpelledWord::touching( 365 AST->getSourceManager().getComposedLoc( 366 AST->getSourceManager().getMainFileID(), A.point()), 367 AST->getTokens(), AST->getLangOpts()); 368 if (A.ranges().size()) { 369 llvm::StringRef Want = A.code().slice(A.range().Begin, A.range().End); 370 EXPECT_EQ(Want, SW->Text) << Text; 371 } 372 return SW; 373 } 374 375 protected: 376 SpelledWord word(const char *Text) { 377 auto Result = tryWord(Text); 378 EXPECT_TRUE(Result) << Text; 379 return Result.getValueOr(SpelledWord()); 380 } 381 382 void noWord(const char *Text) { EXPECT_FALSE(tryWord(Text)) << Text; } 383 }; 384 385 TEST_F(SpelledWordsTest, HeuristicBoundaries) { 386 word("// [[^foo]] "); 387 word("// [[f^oo]] "); 388 word("// [[foo^]] "); 389 word("// [[foo^]]+bar "); 390 noWord("//^ foo "); 391 noWord("// foo ^"); 392 } 393 394 TEST_F(SpelledWordsTest, LikelyIdentifier) { 395 EXPECT_FALSE(word("// ^foo ").LikelyIdentifier); 396 EXPECT_TRUE(word("// [[^foo_bar]] ").LikelyIdentifier); 397 EXPECT_TRUE(word("// [[^fooBar]] ").LikelyIdentifier); 398 EXPECT_FALSE(word("// H^TTP ").LikelyIdentifier); 399 EXPECT_TRUE(word("// \\p [[^foo]] ").LikelyIdentifier); 400 EXPECT_TRUE(word("// @param[in] [[^foo]] ").LikelyIdentifier); 401 EXPECT_TRUE(word("// `[[f^oo]]` ").LikelyIdentifier); 402 EXPECT_TRUE(word("// bar::[[f^oo]] ").LikelyIdentifier); 403 EXPECT_TRUE(word("// [[f^oo]]::bar ").LikelyIdentifier); 404 } 405 406 TEST_F(SpelledWordsTest, Comment) { 407 auto W = word("// [[^foo]]"); 408 EXPECT_FALSE(W.PartOfSpelledToken); 409 EXPECT_FALSE(W.SpelledToken); 410 EXPECT_FALSE(W.ExpandedToken); 411 } 412 413 TEST_F(SpelledWordsTest, PartOfString) { 414 auto W = word(R"( auto str = "foo [[^bar]] baz"; )"); 415 ASSERT_TRUE(W.PartOfSpelledToken); 416 EXPECT_EQ(W.PartOfSpelledToken->kind(), tok::string_literal); 417 EXPECT_FALSE(W.SpelledToken); 418 EXPECT_FALSE(W.ExpandedToken); 419 } 420 421 TEST_F(SpelledWordsTest, DisabledSection) { 422 auto W = word(R"cpp( 423 #if 0 424 foo [[^bar]] baz 425 #endif 426 )cpp"); 427 ASSERT_TRUE(W.SpelledToken); 428 EXPECT_EQ(W.SpelledToken->kind(), tok::identifier); 429 EXPECT_EQ(W.SpelledToken, W.PartOfSpelledToken); 430 EXPECT_FALSE(W.ExpandedToken); 431 } 432 433 TEST_F(SpelledWordsTest, Macros) { 434 auto W = word(R"cpp( 435 #define ID(X) X 436 ID(int [[^i]]); 437 )cpp"); 438 ASSERT_TRUE(W.SpelledToken); 439 EXPECT_EQ(W.SpelledToken->kind(), tok::identifier); 440 EXPECT_EQ(W.SpelledToken, W.PartOfSpelledToken); 441 ASSERT_TRUE(W.ExpandedToken); 442 EXPECT_EQ(W.ExpandedToken->kind(), tok::identifier); 443 444 W = word(R"cpp( 445 #define OBJECT Expansion; 446 int [[^OBJECT]]; 447 )cpp"); 448 EXPECT_TRUE(W.SpelledToken); 449 EXPECT_FALSE(W.ExpandedToken) << "Expanded token is spelled differently"; 450 } 451 452 TEST(SourceCodeTests, VisibleNamespaces) { 453 std::vector<std::pair<const char *, std::vector<std::string>>> Cases = { 454 { 455 R"cpp( 456 // Using directive resolved against enclosing namespaces. 457 using namespace foo; 458 namespace ns { 459 using namespace bar; 460 )cpp", 461 {"ns", "", "bar", "foo", "ns::bar"}, 462 }, 463 { 464 R"cpp( 465 // Don't include namespaces we've closed, ignore namespace aliases. 466 using namespace clang; 467 using std::swap; 468 namespace clang { 469 namespace clangd {} 470 namespace ll = ::llvm; 471 } 472 namespace clang { 473 )cpp", 474 {"clang", ""}, 475 }, 476 { 477 R"cpp( 478 // Using directives visible even if a namespace is reopened. 479 // Ignore anonymous namespaces. 480 namespace foo{ using namespace bar; } 481 namespace foo{ namespace { 482 )cpp", 483 {"foo", "", "bar", "foo::bar"}, 484 }, 485 { 486 R"cpp( 487 // Mismatched braces 488 namespace foo{} 489 }}} 490 namespace bar{ 491 )cpp", 492 {"bar", ""}, 493 }, 494 { 495 R"cpp( 496 // Namespaces with multiple chunks. 497 namespace a::b { 498 using namespace c::d; 499 namespace e::f { 500 )cpp", 501 { 502 "a::b::e::f", 503 "", 504 "a", 505 "a::b", 506 "a::b::c::d", 507 "a::b::e", 508 "a::c::d", 509 "c::d", 510 }, 511 }, 512 { 513 "", 514 {""}, 515 }, 516 { 517 R"cpp( 518 // Parse until EOF 519 namespace bar{})cpp", 520 {""}, 521 }, 522 }; 523 for (const auto &Case : Cases) { 524 EXPECT_EQ(Case.second, 525 visibleNamespaces(Case.first, format::getFormattingLangOpts( 526 format::getLLVMStyle()))) 527 << Case.first; 528 } 529 } 530 531 TEST(SourceCodeTests, GetMacros) { 532 Annotations Code(R"cpp( 533 #define MACRO 123 534 int abc = MA^CRO; 535 )cpp"); 536 TestTU TU = TestTU::withCode(Code.code()); 537 auto AST = TU.build(); 538 auto CurLoc = sourceLocationInMainFile(AST.getSourceManager(), Code.point()); 539 ASSERT_TRUE(bool(CurLoc)); 540 const auto *Id = syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens()); 541 ASSERT_TRUE(Id); 542 auto Result = locateMacroAt(*Id, AST.getPreprocessor()); 543 ASSERT_TRUE(Result); 544 EXPECT_THAT(*Result, macroName("MACRO")); 545 } 546 547 TEST(SourceCodeTests, WorksAtBeginOfFile) { 548 Annotations Code("^MACRO"); 549 TestTU TU = TestTU::withCode(Code.code()); 550 TU.HeaderCode = "#define MACRO int x;"; 551 auto AST = TU.build(); 552 auto CurLoc = sourceLocationInMainFile(AST.getSourceManager(), Code.point()); 553 ASSERT_TRUE(bool(CurLoc)); 554 const auto *Id = syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens()); 555 ASSERT_TRUE(Id); 556 auto Result = locateMacroAt(*Id, AST.getPreprocessor()); 557 ASSERT_TRUE(Result); 558 EXPECT_THAT(*Result, macroName("MACRO")); 559 } 560 561 TEST(SourceCodeTests, IsInsideMainFile) { 562 TestTU TU; 563 TU.HeaderCode = R"cpp( 564 #define DEFINE_CLASS(X) class X {}; 565 #define DEFINE_YY DEFINE_CLASS(YY) 566 567 class Header1 {}; 568 DEFINE_CLASS(Header2) 569 class Header {}; 570 )cpp"; 571 TU.Code = R"cpp( 572 #define DEFINE_MAIN4 class Main4{}; 573 class Main1 {}; 574 DEFINE_CLASS(Main2) 575 DEFINE_YY 576 class Main {}; 577 DEFINE_MAIN4 578 )cpp"; 579 TU.ExtraArgs.push_back("-DHeader=Header3"); 580 TU.ExtraArgs.push_back("-DMain=Main3"); 581 auto AST = TU.build(); 582 const auto &SM = AST.getSourceManager(); 583 auto DeclLoc = [&AST](llvm::StringRef Name) { 584 return findDecl(AST, Name).getLocation(); 585 }; 586 for (const auto *HeaderDecl : {"Header1", "Header2", "Header3"}) 587 EXPECT_FALSE(isInsideMainFile(DeclLoc(HeaderDecl), SM)) << HeaderDecl; 588 589 for (const auto *MainDecl : {"Main1", "Main2", "Main3", "Main4", "YY"}) 590 EXPECT_TRUE(isInsideMainFile(DeclLoc(MainDecl), SM)) << MainDecl; 591 592 // Main4 is *spelled* in the preamble, but in the main-file part of it. 593 EXPECT_TRUE(isInsideMainFile(SM.getSpellingLoc(DeclLoc("Main4")), SM)); 594 } 595 596 // Test for functions toHalfOpenFileRange and getHalfOpenFileRange 597 TEST(SourceCodeTests, HalfOpenFileRange) { 598 // Each marked range should be the file range of the decl with the same name 599 // and each name should be unique. 600 Annotations Test(R"cpp( 601 #define FOO(X, Y) int Y = ++X 602 #define BAR(X) X + 1 603 #define ECHO(X) X 604 605 #define BUZZ BAZZ(ADD) 606 #define BAZZ(m) m(1) 607 #define ADD(a) int f = a + 1; 608 template<typename T> 609 class P {}; 610 611 int main() { 612 $a[[P<P<P<P<P<int>>>>> a]]; 613 $b[[int b = 1]]; 614 $c[[FOO(b, c)]]; 615 $d[[FOO(BAR(BAR(b)), d)]]; 616 // FIXME: We might want to select everything inside the outer ECHO. 617 ECHO(ECHO($e[[int) ECHO(e]])); 618 // Shouldn't crash. 619 $f[[BUZZ]]; 620 } 621 )cpp"); 622 623 ParsedAST AST = TestTU::withCode(Test.code()).build(); 624 llvm::errs() << Test.code(); 625 const SourceManager &SM = AST.getSourceManager(); 626 const LangOptions &LangOpts = AST.getLangOpts(); 627 // Turn a SourceLocation into a pair of positions 628 auto SourceRangeToRange = [&SM](SourceRange SrcRange) { 629 return Range{sourceLocToPosition(SM, SrcRange.getBegin()), 630 sourceLocToPosition(SM, SrcRange.getEnd())}; 631 }; 632 auto CheckRange = [&](llvm::StringRef Name) { 633 const NamedDecl &Decl = findUnqualifiedDecl(AST, Name); 634 auto FileRange = toHalfOpenFileRange(SM, LangOpts, Decl.getSourceRange()); 635 SCOPED_TRACE("Checking range: " + Name); 636 ASSERT_NE(FileRange, llvm::None); 637 Range HalfOpenRange = SourceRangeToRange(*FileRange); 638 EXPECT_EQ(HalfOpenRange, Test.ranges(Name)[0]); 639 }; 640 641 CheckRange("a"); 642 CheckRange("b"); 643 CheckRange("c"); 644 CheckRange("d"); 645 CheckRange("e"); 646 CheckRange("f"); 647 } 648 649 TEST(SourceCodeTests, HalfOpenFileRangePathologicalPreprocessor) { 650 const char *Case = R"cpp( 651 #define MACRO while(1) 652 void test() { 653 [[#include "Expand.inc" 654 br^eak]]; 655 } 656 )cpp"; 657 Annotations Test(Case); 658 auto TU = TestTU::withCode(Test.code()); 659 TU.AdditionalFiles["Expand.inc"] = "MACRO\n"; 660 auto AST = TU.build(); 661 662 const auto &Func = cast<FunctionDecl>(findDecl(AST, "test")); 663 const auto &Body = cast<CompoundStmt>(Func.getBody()); 664 const auto &Loop = cast<WhileStmt>(*Body->child_begin()); 665 llvm::Optional<SourceRange> Range = toHalfOpenFileRange( 666 AST.getSourceManager(), AST.getLangOpts(), Loop->getSourceRange()); 667 ASSERT_TRUE(Range) << "Failed to get file range"; 668 EXPECT_EQ(AST.getSourceManager().getFileOffset(Range->getBegin()), 669 Test.llvm::Annotations::range().Begin); 670 EXPECT_EQ(AST.getSourceManager().getFileOffset(Range->getEnd()), 671 Test.llvm::Annotations::range().End); 672 } 673 674 TEST(SourceCodeTests, IncludeHashLoc) { 675 const char *Case = R"cpp( 676 $foo^#include "foo.inc" 677 #define HEADER "bar.inc" 678 $bar^# include HEADER 679 )cpp"; 680 Annotations Test(Case); 681 auto TU = TestTU::withCode(Test.code()); 682 TU.AdditionalFiles["foo.inc"] = "int foo;\n"; 683 TU.AdditionalFiles["bar.inc"] = "int bar;\n"; 684 auto AST = TU.build(); 685 const auto &SM = AST.getSourceManager(); 686 687 FileID Foo = SM.getFileID(findDecl(AST, "foo").getLocation()); 688 EXPECT_EQ(SM.getFileOffset(includeHashLoc(Foo, SM)), 689 Test.llvm::Annotations::point("foo")); 690 FileID Bar = SM.getFileID(findDecl(AST, "bar").getLocation()); 691 EXPECT_EQ(SM.getFileOffset(includeHashLoc(Bar, SM)), 692 Test.llvm::Annotations::point("bar")); 693 } 694 695 TEST(SourceCodeTests, GetEligiblePoints) { 696 constexpr struct { 697 const char *Code; 698 const char *FullyQualifiedName; 699 const char *EnclosingNamespace; 700 } Cases[] = { 701 {R"cpp(// FIXME: We should also mark positions before and after 702 //declarations/definitions as eligible. 703 namespace ns1 { 704 namespace a { namespace ns2 {} } 705 namespace ns2 {^ 706 void foo(); 707 namespace {} 708 void bar() {} 709 namespace ns3 {} 710 class T {}; 711 ^} 712 using namespace ns2; 713 })cpp", 714 "ns1::ns2::symbol", "ns1::ns2::"}, 715 {R"cpp( 716 namespace ns1 {^ 717 namespace a { namespace ns2 {} } 718 namespace b {} 719 namespace ns {} 720 ^})cpp", 721 "ns1::ns2::symbol", "ns1::"}, 722 {R"cpp( 723 namespace x { 724 namespace a { namespace ns2 {} } 725 namespace b {} 726 namespace ns {} 727 }^)cpp", 728 "ns1::ns2::symbol", ""}, 729 {R"cpp( 730 namespace ns1 { 731 namespace ns2 {^^} 732 namespace b {} 733 namespace ns2 {^^} 734 } 735 namespace ns1 {namespace ns2 {^^}})cpp", 736 "ns1::ns2::symbol", "ns1::ns2::"}, 737 {R"cpp( 738 namespace ns1 {^ 739 namespace ns {} 740 namespace b {} 741 namespace ns {} 742 ^} 743 namespace ns1 {^namespace ns {}^})cpp", 744 "ns1::ns2::symbol", "ns1::"}, 745 }; 746 for (auto Case : Cases) { 747 Annotations Test(Case.Code); 748 749 auto Res = getEligiblePoints( 750 Test.code(), Case.FullyQualifiedName, 751 format::getFormattingLangOpts(format::getLLVMStyle())); 752 EXPECT_THAT(Res.EligiblePoints, testing::ElementsAreArray(Test.points())) 753 << Test.code(); 754 EXPECT_EQ(Res.EnclosingNamespace, Case.EnclosingNamespace) << Test.code(); 755 } 756 } 757 758 TEST(SourceCodeTests, IdentifierRanges) { 759 Annotations Code(R"cpp( 760 class [[Foo]] {}; 761 // Foo 762 /* Foo */ 763 void f([[Foo]]* foo1) { 764 [[Foo]] foo2; 765 auto S = [[Foo]](); 766 // cross-line identifier is not supported. 767 F\ 768 o\ 769 o foo2; 770 } 771 )cpp"); 772 LangOptions LangOpts; 773 LangOpts.CPlusPlus = true; 774 EXPECT_EQ(Code.ranges(), 775 collectIdentifierRanges("Foo", Code.code(), LangOpts)); 776 } 777 778 TEST(SourceCodeTests, isHeaderFile) { 779 // Without lang options. 780 EXPECT_TRUE(isHeaderFile("foo.h")); 781 EXPECT_TRUE(isHeaderFile("foo.hh")); 782 EXPECT_TRUE(isHeaderFile("foo.hpp")); 783 784 EXPECT_FALSE(isHeaderFile("foo.cpp")); 785 EXPECT_FALSE(isHeaderFile("foo.c++")); 786 EXPECT_FALSE(isHeaderFile("foo.cxx")); 787 EXPECT_FALSE(isHeaderFile("foo.cc")); 788 EXPECT_FALSE(isHeaderFile("foo.c")); 789 EXPECT_FALSE(isHeaderFile("foo.mm")); 790 EXPECT_FALSE(isHeaderFile("foo.m")); 791 792 // With lang options 793 LangOptions LangOpts; 794 LangOpts.IsHeaderFile = true; 795 EXPECT_TRUE(isHeaderFile("string", LangOpts)); 796 // Emulate cases where there is no "-x header" flag for a .h file, we still 797 // want to treat it as a header. 798 LangOpts.IsHeaderFile = false; 799 EXPECT_TRUE(isHeaderFile("header.h", LangOpts)); 800 } 801 802 TEST(SourceCodeTests, isKeywords) { 803 LangOptions LangOpts; 804 LangOpts.CPlusPlus20 = true; 805 EXPECT_TRUE(isKeyword("int", LangOpts)); 806 EXPECT_TRUE(isKeyword("return", LangOpts)); 807 EXPECT_TRUE(isKeyword("co_await", LangOpts)); 808 809 // these are identifiers (not keywords!) with special meaning in some 810 // contexts. 811 EXPECT_FALSE(isKeyword("final", LangOpts)); 812 EXPECT_FALSE(isKeyword("override", LangOpts)); 813 } 814 815 struct IncrementalTestStep { 816 llvm::StringRef Src; 817 llvm::StringRef Contents; 818 }; 819 820 int rangeLength(llvm::StringRef Code, const Range &Rng) { 821 llvm::Expected<size_t> Start = positionToOffset(Code, Rng.start); 822 llvm::Expected<size_t> End = positionToOffset(Code, Rng.end); 823 assert(Start); 824 assert(End); 825 return *End - *Start; 826 } 827 828 /// Send the changes one by one to updateDraft, verify the intermediate results. 829 void stepByStep(llvm::ArrayRef<IncrementalTestStep> Steps) { 830 std::string Code = Annotations(Steps.front().Src).code().str(); 831 832 for (size_t I = 1; I < Steps.size(); I++) { 833 Annotations SrcBefore(Steps[I - 1].Src); 834 Annotations SrcAfter(Steps[I].Src); 835 llvm::StringRef Contents = Steps[I - 1].Contents; 836 TextDocumentContentChangeEvent Event{ 837 SrcBefore.range(), 838 rangeLength(SrcBefore.code(), SrcBefore.range()), 839 Contents.str(), 840 }; 841 842 EXPECT_THAT_ERROR(applyChange(Code, Event), llvm::Succeeded()); 843 EXPECT_EQ(Code, SrcAfter.code()); 844 } 845 } 846 847 TEST(ApplyEditsTest, Simple) { 848 // clang-format off 849 IncrementalTestStep Steps[] = 850 { 851 // Replace a range 852 { 853 R"cpp(static int 854 hello[[World]]() 855 {})cpp", 856 "Universe" 857 }, 858 // Delete a range 859 { 860 R"cpp(static int 861 hello[[Universe]]() 862 {})cpp", 863 "" 864 }, 865 // Add a range 866 { 867 R"cpp(static int 868 hello[[]]() 869 {})cpp", 870 "Monde" 871 }, 872 { 873 R"cpp(static int 874 helloMonde() 875 {})cpp", 876 "" 877 } 878 }; 879 // clang-format on 880 881 stepByStep(Steps); 882 } 883 884 TEST(ApplyEditsTest, MultiLine) { 885 // clang-format off 886 IncrementalTestStep Steps[] = 887 { 888 // Replace a range 889 { 890 R"cpp(static [[int 891 helloWorld]]() 892 {})cpp", 893 R"cpp(char 894 welcome)cpp" 895 }, 896 // Delete a range 897 { 898 R"cpp(static char[[ 899 welcome]]() 900 {})cpp", 901 "" 902 }, 903 // Add a range 904 { 905 R"cpp(static char[[]]() 906 {})cpp", 907 R"cpp( 908 cookies)cpp" 909 }, 910 // Replace the whole file 911 { 912 R"cpp([[static char 913 cookies() 914 {}]])cpp", 915 R"cpp(#include <stdio.h> 916 )cpp" 917 }, 918 // Delete the whole file 919 { 920 R"cpp([[#include <stdio.h> 921 ]])cpp", 922 "", 923 }, 924 // Add something to an empty file 925 { 926 "[[]]", 927 R"cpp(int main() { 928 )cpp", 929 }, 930 { 931 R"cpp(int main() { 932 )cpp", 933 "" 934 } 935 }; 936 // clang-format on 937 938 stepByStep(Steps); 939 } 940 941 TEST(ApplyEditsTest, WrongRangeLength) { 942 std::string Code = "int main() {}\n"; 943 944 TextDocumentContentChangeEvent Change; 945 Change.range.emplace(); 946 Change.range->start.line = 0; 947 Change.range->start.character = 0; 948 Change.range->end.line = 0; 949 Change.range->end.character = 2; 950 Change.rangeLength = 10; 951 952 EXPECT_THAT_ERROR(applyChange(Code, Change), 953 FailedWithMessage("Change's rangeLength (10) doesn't match " 954 "the computed range length (2).")); 955 } 956 957 TEST(ApplyEditsTest, EndBeforeStart) { 958 std::string Code = "int main() {}\n"; 959 960 TextDocumentContentChangeEvent Change; 961 Change.range.emplace(); 962 Change.range->start.line = 0; 963 Change.range->start.character = 5; 964 Change.range->end.line = 0; 965 Change.range->end.character = 3; 966 967 EXPECT_THAT_ERROR( 968 applyChange(Code, Change), 969 FailedWithMessage( 970 "Range's end position (0:3) is before start position (0:5)")); 971 } 972 973 TEST(ApplyEditsTest, StartCharOutOfRange) { 974 std::string Code = "int main() {}\n"; 975 976 TextDocumentContentChangeEvent Change; 977 Change.range.emplace(); 978 Change.range->start.line = 0; 979 Change.range->start.character = 100; 980 Change.range->end.line = 0; 981 Change.range->end.character = 100; 982 Change.text = "foo"; 983 984 EXPECT_THAT_ERROR( 985 applyChange(Code, Change), 986 FailedWithMessage("utf-16 offset 100 is invalid for line 0")); 987 } 988 989 TEST(ApplyEditsTest, EndCharOutOfRange) { 990 std::string Code = "int main() {}\n"; 991 992 TextDocumentContentChangeEvent Change; 993 Change.range.emplace(); 994 Change.range->start.line = 0; 995 Change.range->start.character = 0; 996 Change.range->end.line = 0; 997 Change.range->end.character = 100; 998 Change.text = "foo"; 999 1000 EXPECT_THAT_ERROR( 1001 applyChange(Code, Change), 1002 FailedWithMessage("utf-16 offset 100 is invalid for line 0")); 1003 } 1004 1005 TEST(ApplyEditsTest, StartLineOutOfRange) { 1006 std::string Code = "int main() {}\n"; 1007 1008 TextDocumentContentChangeEvent Change; 1009 Change.range.emplace(); 1010 Change.range->start.line = 100; 1011 Change.range->start.character = 0; 1012 Change.range->end.line = 100; 1013 Change.range->end.character = 0; 1014 Change.text = "foo"; 1015 1016 EXPECT_THAT_ERROR(applyChange(Code, Change), 1017 FailedWithMessage("Line value is out of range (100)")); 1018 } 1019 1020 TEST(ApplyEditsTest, EndLineOutOfRange) { 1021 std::string Code = "int main() {}\n"; 1022 1023 TextDocumentContentChangeEvent Change; 1024 Change.range.emplace(); 1025 Change.range->start.line = 0; 1026 Change.range->start.character = 0; 1027 Change.range->end.line = 100; 1028 Change.range->end.character = 0; 1029 Change.text = "foo"; 1030 1031 EXPECT_THAT_ERROR(applyChange(Code, Change), 1032 FailedWithMessage("Line value is out of range (100)")); 1033 } 1034 1035 } // namespace 1036 } // namespace clangd 1037 } // namespace clang 1038