1 //===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Format/Format.h" 10 11 #include "FormatTestUtils.h" 12 #include "TestLexer.h" 13 #include "gtest/gtest.h" 14 15 namespace clang { 16 namespace format { 17 18 // Not really the equality, but everything we need. 19 static bool operator==(const FormatToken &LHS, 20 const FormatToken &RHS) noexcept { 21 return LHS.Tok.getKind() == RHS.Tok.getKind() && 22 LHS.getType() == RHS.getType(); 23 } 24 25 namespace { 26 27 class TokenAnnotatorTest : public ::testing::Test { 28 protected: 29 TokenList annotate(llvm::StringRef Code, 30 const FormatStyle &Style = getLLVMStyle()) { 31 return TestLexer(Allocator, Buffers, Style).annotate(Code); 32 } 33 llvm::SpecificBumpPtrAllocator<FormatToken> Allocator; 34 std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers; 35 }; 36 37 #define EXPECT_TOKEN_KIND(FormatTok, Kind) \ 38 EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok) 39 #define EXPECT_TOKEN_TYPE(FormatTok, Type) \ 40 EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok) 41 #define EXPECT_TOKEN(FormatTok, Kind, Type) \ 42 do { \ 43 EXPECT_TOKEN_KIND(FormatTok, Kind); \ 44 EXPECT_TOKEN_TYPE(FormatTok, Type); \ 45 } while (false); 46 47 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmpInMacroDefinition) { 48 // This is a regression test for mis-parsing the & after decltype as a binary 49 // operator instead of a reference (when inside a macro definition). 50 auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};"); 51 EXPECT_EQ(Tokens.size(), 18u) << Tokens; 52 EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown); 53 EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen); 54 EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown); 55 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 56 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); 57 // Same again with * instead of &: 58 Tokens = annotate("auto x = [](const decltype(x) *ptr) {};"); 59 EXPECT_EQ(Tokens.size(), 18u) << Tokens; 60 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 61 EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference); 62 63 // Also check that we parse correctly within a macro definition: 64 Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}"); 65 EXPECT_EQ(Tokens.size(), 17u) << Tokens; 66 EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown); 67 EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen); 68 EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown); 69 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 70 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); 71 // Same again with * instead of &: 72 Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}"); 73 EXPECT_EQ(Tokens.size(), 17u) << Tokens; 74 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 75 EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference); 76 } 77 78 TEST_F(TokenAnnotatorTest, UnderstandsClasses) { 79 auto Tokens = annotate("class C {};"); 80 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 81 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_RecordLBrace); 82 } 83 84 TEST_F(TokenAnnotatorTest, UnderstandsStructs) { 85 auto Tokens = annotate("struct S {};"); 86 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 87 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_RecordLBrace); 88 } 89 90 TEST_F(TokenAnnotatorTest, UnderstandsUnions) { 91 auto Tokens = annotate("union U {};"); 92 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 93 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_RecordLBrace); 94 } 95 96 TEST_F(TokenAnnotatorTest, UnderstandsEnums) { 97 auto Tokens = annotate("enum E {};"); 98 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 99 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_RecordLBrace); 100 } 101 102 TEST_F(TokenAnnotatorTest, UnderstandsLBracesInMacroDefinition) { 103 auto Tokens = annotate("#define BEGIN NS {"); 104 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 105 EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); 106 } 107 108 TEST_F(TokenAnnotatorTest, UnderstandsDelete) { 109 auto Tokens = annotate("delete (void *)p;"); 110 EXPECT_EQ(Tokens.size(), 8u) << Tokens; 111 EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen); 112 113 Tokens = annotate("delete[] (void *)p;"); 114 EXPECT_EQ(Tokens.size(), 10u) << Tokens; 115 EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen); 116 117 Tokens = annotate("delete[] /*comment*/ (void *)p;"); 118 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 119 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 120 121 Tokens = annotate("delete[/*comment*/] (void *)p;"); 122 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 123 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 124 125 Tokens = annotate("delete/*comment*/[] (void *)p;"); 126 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 127 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 128 } 129 130 TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) { 131 auto Tokens = annotate("template <typename T>\n" 132 "concept C = (Foo && Bar) && (Bar && Baz);"); 133 134 ASSERT_EQ(Tokens.size(), 21u) << Tokens; 135 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator); 136 EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator); 137 EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator); 138 139 Tokens = annotate("template <typename T>\n" 140 "concept C = requires(T t) {\n" 141 " { t.foo() };\n" 142 "} && Bar<T> && Baz<T>;"); 143 ASSERT_EQ(Tokens.size(), 35u) << Tokens; 144 EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator); 145 EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator); 146 147 Tokens = annotate("template<typename T>\n" 148 "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n" 149 "struct Foo;"); 150 ASSERT_EQ(Tokens.size(), 36u) << Tokens; 151 EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown); 152 EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u); 153 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator); 154 EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator); 155 EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator); 156 EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator); 157 EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser); 158 EXPECT_EQ(Tokens[31]->FakeRParens, 1u); 159 EXPECT_TRUE(Tokens[31]->ClosesRequiresClause); 160 161 Tokens = 162 annotate("template<typename T>\n" 163 "requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n" 164 "struct Foo;"); 165 ASSERT_EQ(Tokens.size(), 38u) << Tokens; 166 EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown); 167 EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u); 168 EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator); 169 EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator); 170 EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator); 171 EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator); 172 EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser); 173 EXPECT_EQ(Tokens[32]->FakeRParens, 1u); 174 EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown); 175 EXPECT_TRUE(Tokens[33]->ClosesRequiresClause); 176 } 177 178 TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) { 179 auto NumberOfAdditionalRequiresClauseTokens = 5u; 180 auto NumberOfTokensBeforeRequires = 5u; 181 182 auto BaseTokens = annotate("template<typename T>\n" 183 "T Pi = 3.14;"); 184 auto ConstrainedTokens = annotate("template<typename T>\n" 185 " requires Foo<T>\n" 186 "T Pi = 3.14;"); 187 188 auto NumberOfBaseTokens = 11u; 189 190 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 191 ASSERT_EQ(ConstrainedTokens.size(), 192 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 193 << ConstrainedTokens; 194 195 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 196 if (I < NumberOfTokensBeforeRequires) 197 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 198 else 199 EXPECT_EQ(*BaseTokens[I], 200 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 201 << I; 202 203 BaseTokens = annotate("template<typename T>\n" 204 "struct Bar;"); 205 ConstrainedTokens = annotate("template<typename T>\n" 206 " requires Foo<T>\n" 207 "struct Bar;"); 208 NumberOfBaseTokens = 9u; 209 210 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 211 ASSERT_EQ(ConstrainedTokens.size(), 212 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 213 << ConstrainedTokens; 214 215 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 216 if (I < NumberOfTokensBeforeRequires) 217 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 218 else 219 EXPECT_EQ(*BaseTokens[I], 220 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 221 << I; 222 223 BaseTokens = annotate("template<typename T>\n" 224 "struct Bar {" 225 " T foo();\n" 226 " T bar();\n" 227 "};"); 228 ConstrainedTokens = annotate("template<typename T>\n" 229 " requires Foo<T>\n" 230 "struct Bar {" 231 " T foo();\n" 232 " T bar();\n" 233 "};"); 234 NumberOfBaseTokens = 21u; 235 236 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 237 ASSERT_EQ(ConstrainedTokens.size(), 238 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 239 << ConstrainedTokens; 240 241 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 242 if (I < NumberOfTokensBeforeRequires) 243 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 244 else 245 EXPECT_EQ(*BaseTokens[I], 246 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 247 << I; 248 249 BaseTokens = annotate("template<typename T>\n" 250 "Bar(T) -> Bar<T>;"); 251 ConstrainedTokens = annotate("template<typename T>\n" 252 " requires Foo<T>\n" 253 "Bar(T) -> Bar<T>;"); 254 NumberOfBaseTokens = 16u; 255 256 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 257 ASSERT_EQ(ConstrainedTokens.size(), 258 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 259 << ConstrainedTokens; 260 261 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 262 if (I < NumberOfTokensBeforeRequires) 263 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 264 else 265 EXPECT_EQ(*BaseTokens[I], 266 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 267 << I; 268 269 BaseTokens = annotate("template<typename T>\n" 270 "T foo();"); 271 ConstrainedTokens = annotate("template<typename T>\n" 272 " requires Foo<T>\n" 273 "T foo();"); 274 NumberOfBaseTokens = 11u; 275 276 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 277 ASSERT_EQ(ConstrainedTokens.size(), 278 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 279 << ConstrainedTokens; 280 281 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 282 if (I < NumberOfTokensBeforeRequires) 283 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 284 else 285 EXPECT_EQ(*BaseTokens[I], 286 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 287 << I; 288 289 BaseTokens = annotate("template<typename T>\n" 290 "T foo() {\n" 291 " auto bar = baz();\n" 292 " return bar + T{};\n" 293 "}"); 294 ConstrainedTokens = annotate("template<typename T>\n" 295 " requires Foo<T>\n" 296 "T foo() {\n" 297 " auto bar = baz();\n" 298 " return bar + T{};\n" 299 "}"); 300 NumberOfBaseTokens = 26u; 301 302 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 303 ASSERT_EQ(ConstrainedTokens.size(), 304 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 305 << ConstrainedTokens; 306 307 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 308 if (I < NumberOfTokensBeforeRequires) 309 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 310 else 311 EXPECT_EQ(*BaseTokens[I], 312 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 313 << I; 314 315 BaseTokens = annotate("template<typename T>\n" 316 "T foo();"); 317 ConstrainedTokens = annotate("template<typename T>\n" 318 "T foo() requires Foo<T>;"); 319 NumberOfBaseTokens = 11u; 320 NumberOfTokensBeforeRequires = 9u; 321 322 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 323 ASSERT_EQ(ConstrainedTokens.size(), 324 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 325 << ConstrainedTokens; 326 327 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 328 if (I < NumberOfTokensBeforeRequires) 329 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 330 else 331 EXPECT_EQ(*BaseTokens[I], 332 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 333 << I; 334 335 BaseTokens = annotate("template<typename T>\n" 336 "T foo() {\n" 337 " auto bar = baz();\n" 338 " return bar + T{};\n" 339 "}"); 340 ConstrainedTokens = annotate("template<typename T>\n" 341 "T foo() requires Foo<T> {\n" 342 " auto bar = baz();\n" 343 " return bar + T{};\n" 344 "}"); 345 NumberOfBaseTokens = 26u; 346 347 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 348 ASSERT_EQ(ConstrainedTokens.size(), 349 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 350 << ConstrainedTokens; 351 352 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 353 if (I < NumberOfTokensBeforeRequires) 354 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 355 else 356 EXPECT_EQ(*BaseTokens[I], 357 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 358 << I; 359 360 BaseTokens = annotate("template<typename T>\n" 361 "Bar(T) -> Bar<typename T::I>;"); 362 ConstrainedTokens = annotate("template<typename T>\n" 363 " requires requires(T &&t) {\n" 364 " typename T::I;\n" 365 " }\n" 366 "Bar(T) -> Bar<typename T::I>;"); 367 NumberOfBaseTokens = 19u; 368 NumberOfAdditionalRequiresClauseTokens = 14u; 369 NumberOfTokensBeforeRequires = 5u; 370 371 ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens; 372 ASSERT_EQ(ConstrainedTokens.size(), 373 NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens) 374 << ConstrainedTokens; 375 376 for (auto I = 0u; I < NumberOfBaseTokens; ++I) 377 if (I < NumberOfTokensBeforeRequires) 378 EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I; 379 else 380 EXPECT_EQ(*BaseTokens[I], 381 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens]) 382 << I; 383 } 384 385 } // namespace 386 } // namespace format 387 } // namespace clang 388