1 //===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Format/Format.h" 10 11 #include "FormatTestUtils.h" 12 #include "TestLexer.h" 13 #include "gtest/gtest.h" 14 15 namespace clang { 16 namespace format { 17 18 // Not really the equality, but everything we need. 19 static bool operator==(const FormatToken &LHS, 20 const FormatToken &RHS) noexcept { 21 return LHS.Tok.getKind() == RHS.Tok.getKind() && 22 LHS.getType() == RHS.getType(); 23 } 24 25 namespace { 26 27 class TokenAnnotatorTest : public ::testing::Test { 28 protected: 29 TokenList annotate(llvm::StringRef Code, 30 const FormatStyle &Style = getLLVMStyle()) { 31 return TestLexer(Allocator, Buffers, Style).annotate(Code); 32 } 33 llvm::SpecificBumpPtrAllocator<FormatToken> Allocator; 34 std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers; 35 }; 36 37 #define EXPECT_TOKEN_KIND(FormatTok, Kind) \ 38 EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok) 39 #define EXPECT_TOKEN_TYPE(FormatTok, Type) \ 40 EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok) 41 #define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec) \ 42 EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok) 43 #define EXPECT_TOKEN(FormatTok, Kind, Type) \ 44 do { \ 45 EXPECT_TOKEN_KIND(FormatTok, Kind); \ 46 EXPECT_TOKEN_TYPE(FormatTok, Type); \ 47 } while (false) 48 49 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) { 50 auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};"); 51 EXPECT_EQ(Tokens.size(), 18u) << Tokens; 52 EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown); 53 EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen); 54 EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown); 55 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 56 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); 57 58 Tokens = annotate("auto x = [](const decltype(x) *ptr) {};"); 59 EXPECT_EQ(Tokens.size(), 18u) << Tokens; 60 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 61 EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference); 62 63 Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}"); 64 EXPECT_EQ(Tokens.size(), 17u) << Tokens; 65 EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown); 66 EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen); 67 EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown); 68 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 69 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); 70 71 Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}"); 72 EXPECT_EQ(Tokens.size(), 17u) << Tokens; 73 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen); 74 EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference); 75 76 Tokens = annotate("void f() {\n" 77 " while (p < a && *p == 'a')\n" 78 " p++;\n" 79 "}"); 80 EXPECT_EQ(Tokens.size(), 21u) << Tokens; 81 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator); 82 EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator); 83 84 Tokens = annotate("case *x:"); 85 EXPECT_EQ(Tokens.size(), 5u) << Tokens; 86 EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator); 87 Tokens = annotate("case &x:"); 88 EXPECT_EQ(Tokens.size(), 5u) << Tokens; 89 EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator); 90 91 Tokens = annotate("bool b = 3 == int{3} && true;\n"); 92 EXPECT_EQ(Tokens.size(), 13u) << Tokens; 93 EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator); 94 95 Tokens = annotate("struct {\n" 96 "} *ptr;"); 97 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 98 EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference); 99 Tokens = annotate("union {\n" 100 "} *ptr;"); 101 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 102 EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference); 103 Tokens = annotate("class {\n" 104 "} *ptr;"); 105 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 106 EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference); 107 108 Tokens = annotate("struct {\n" 109 "} &&ptr = {};"); 110 EXPECT_EQ(Tokens.size(), 10u) << Tokens; 111 EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference); 112 Tokens = annotate("union {\n" 113 "} &&ptr = {};"); 114 EXPECT_EQ(Tokens.size(), 10u) << Tokens; 115 EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference); 116 Tokens = annotate("class {\n" 117 "} &&ptr = {};"); 118 EXPECT_EQ(Tokens.size(), 10u) << Tokens; 119 EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference); 120 Tokens = annotate("int i = int{42} * 2;"); 121 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 122 EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator); 123 124 Tokens = annotate("delete[] *ptr;"); 125 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 126 EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator); 127 Tokens = annotate("delete[] **ptr;"); 128 EXPECT_EQ(Tokens.size(), 8u) << Tokens; 129 EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator); 130 EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator); 131 Tokens = annotate("delete[] *(ptr);"); 132 EXPECT_EQ(Tokens.size(), 9u) << Tokens; 133 EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator); 134 } 135 136 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) { 137 auto Tokens = annotate("x - 0"); 138 ASSERT_EQ(Tokens.size(), 4u) << Tokens; 139 EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator); 140 Tokens = annotate("0 + 0"); 141 ASSERT_EQ(Tokens.size(), 4u) << Tokens; 142 EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator); 143 Tokens = annotate("x + +0"); 144 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 145 EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator); 146 Tokens = annotate("x ? -0 : +0"); 147 ASSERT_EQ(Tokens.size(), 8u) << Tokens; 148 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator); 149 EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator); 150 Tokens = annotate("(-0)"); 151 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 152 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 153 Tokens = annotate("0, -0"); 154 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 155 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator); 156 Tokens = annotate("for (; -1;) {\n}"); 157 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 158 EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator); 159 Tokens = annotate("x = -1;"); 160 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 161 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator); 162 Tokens = annotate("x[-1]"); 163 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 164 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator); 165 Tokens = annotate("x = {-1};"); 166 ASSERT_EQ(Tokens.size(), 8u) << Tokens; 167 EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator); 168 Tokens = annotate("case -x:"); 169 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 170 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 171 Tokens = annotate("co_await -x;"); 172 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 173 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 174 Tokens = annotate("co_return -x;"); 175 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 176 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 177 Tokens = annotate("co_yield -x;"); 178 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 179 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 180 Tokens = annotate("delete -x;"); 181 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 182 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 183 Tokens = annotate("return -x;"); 184 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 185 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 186 Tokens = annotate("throw -x;"); 187 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 188 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 189 Tokens = annotate("sizeof -x"); 190 ASSERT_EQ(Tokens.size(), 4u) << Tokens; 191 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 192 Tokens = annotate("co_await +x;"); 193 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 194 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 195 Tokens = annotate("co_return +x;"); 196 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 197 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 198 Tokens = annotate("co_yield +x;"); 199 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 200 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 201 Tokens = annotate("delete +x;"); 202 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 203 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 204 Tokens = annotate("return +x;"); 205 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 206 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 207 Tokens = annotate("throw +x;"); 208 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 209 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 210 Tokens = annotate("sizeof +x"); 211 ASSERT_EQ(Tokens.size(), 4u) << Tokens; 212 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 213 Tokens = annotate("(int)-x"); 214 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 215 EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator); 216 Tokens = annotate("(-x)"); 217 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 218 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator); 219 Tokens = annotate("!+x"); 220 ASSERT_EQ(Tokens.size(), 4u) << Tokens; 221 EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator); 222 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator); 223 } 224 225 TEST_F(TokenAnnotatorTest, UnderstandsClasses) { 226 auto Tokens = annotate("class C {};"); 227 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 228 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace); 229 230 Tokens = annotate("const class C {} c;"); 231 EXPECT_EQ(Tokens.size(), 8u) << Tokens; 232 EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace); 233 234 Tokens = annotate("const class {} c;"); 235 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 236 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace); 237 } 238 239 TEST_F(TokenAnnotatorTest, UnderstandsStructs) { 240 auto Tokens = annotate("struct S {};"); 241 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 242 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace); 243 } 244 245 TEST_F(TokenAnnotatorTest, UnderstandsUnions) { 246 auto Tokens = annotate("union U {};"); 247 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 248 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace); 249 250 Tokens = annotate("union U { void f() { return; } };"); 251 EXPECT_EQ(Tokens.size(), 14u) << Tokens; 252 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace); 253 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace); 254 } 255 256 TEST_F(TokenAnnotatorTest, UnderstandsEnums) { 257 auto Tokens = annotate("enum E {};"); 258 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 259 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace); 260 } 261 262 TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) { 263 auto Tokens = annotate("auto operator<=>(const T &) const & = default;"); 264 EXPECT_EQ(Tokens.size(), 14u) << Tokens; 265 EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference); 266 267 Tokens = annotate("template <typename T> void F(T) && = delete;"); 268 EXPECT_EQ(Tokens.size(), 15u) << Tokens; 269 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference); 270 } 271 272 TEST_F(TokenAnnotatorTest, UnderstandsVariables) { 273 auto Tokens = 274 annotate("inline bool var = is_integral_v<int> && is_signed_v<int>;"); 275 EXPECT_EQ(Tokens.size(), 15u) << Tokens; 276 EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator); 277 } 278 279 TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) { 280 auto Tokens = 281 annotate("template <typename T> " 282 "inline bool var = is_integral_v<int> && is_signed_v<int>;"); 283 EXPECT_EQ(Tokens.size(), 20u) << Tokens; 284 EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator); 285 } 286 287 TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) { 288 FormatStyle Style = getLLVMStyle(); 289 Style.WhitespaceSensitiveMacros.push_back("FOO"); 290 291 auto Tokens = annotate("FOO(1+2 )\n", Style); 292 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 293 EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc); 294 295 Tokens = annotate("FOO(a:b:c)\n", Style); 296 EXPECT_EQ(Tokens.size(), 9u) << Tokens; 297 EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc); 298 } 299 300 TEST_F(TokenAnnotatorTest, UnderstandsDelete) { 301 auto Tokens = annotate("delete (void *)p;"); 302 EXPECT_EQ(Tokens.size(), 8u) << Tokens; 303 EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen); 304 305 Tokens = annotate("delete[] (void *)p;"); 306 EXPECT_EQ(Tokens.size(), 10u) << Tokens; 307 EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen); 308 309 Tokens = annotate("delete[] /*comment*/ (void *)p;"); 310 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 311 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 312 313 Tokens = annotate("delete[/*comment*/] (void *)p;"); 314 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 315 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 316 317 Tokens = annotate("delete/*comment*/[] (void *)p;"); 318 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 319 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 320 } 321 322 TEST_F(TokenAnnotatorTest, UnderstandsCasts) { 323 auto Tokens = annotate("(void)p;"); 324 EXPECT_EQ(Tokens.size(), 6u) << Tokens; 325 EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen); 326 327 Tokens = annotate("auto x = (Foo)p;"); 328 EXPECT_EQ(Tokens.size(), 9u) << Tokens; 329 EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen); 330 331 Tokens = annotate("(std::vector<int>)p;"); 332 EXPECT_EQ(Tokens.size(), 11u) << Tokens; 333 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen); 334 335 Tokens = annotate("return (Foo)p;"); 336 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 337 EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen); 338 339 Tokens = annotate("throw (Foo)p;"); 340 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 341 EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen); 342 } 343 344 TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) { 345 auto Tokens = annotate("void f() throw(int);"); 346 EXPECT_EQ(Tokens.size(), 10u) << Tokens; 347 EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown); 348 } 349 350 TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) { 351 auto Tokens = annotate("void f() &;"); 352 EXPECT_EQ(Tokens.size(), 7u) << Tokens; 353 EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference); 354 355 Tokens = annotate("void operator=(T) &&;"); 356 EXPECT_EQ(Tokens.size(), 9u) << Tokens; 357 EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference); 358 359 Tokens = annotate("template <typename T> void f() &;"); 360 EXPECT_EQ(Tokens.size(), 12u) << Tokens; 361 EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference); 362 363 Tokens = annotate("template <typename T> void operator=(T) &;"); 364 EXPECT_EQ(Tokens.size(), 14u) << Tokens; 365 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); 366 } 367 368 TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) { 369 auto Tokens = annotate("template <typename T>\n" 370 "concept C = (Foo && Bar) && (Bar && Baz);"); 371 372 ASSERT_EQ(Tokens.size(), 21u) << Tokens; 373 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator); 374 EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator); 375 EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator); 376 377 Tokens = annotate("template <typename T>\n" 378 "concept C = Foo && !Bar;"); 379 380 ASSERT_EQ(Tokens.size(), 14u) << Tokens; 381 EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator); 382 EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator); 383 384 Tokens = annotate("template <typename T>\n" 385 "concept C = requires(T t) {\n" 386 " { t.foo() };\n" 387 "} && Bar<T> && Baz<T>;"); 388 ASSERT_EQ(Tokens.size(), 35u) << Tokens; 389 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression); 390 EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen); 391 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace); 392 EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator); 393 EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator); 394 395 Tokens = annotate("template<typename T>\n" 396 "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n" 397 "struct Foo;"); 398 ASSERT_EQ(Tokens.size(), 36u) << Tokens; 399 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause); 400 EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown); 401 EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u); 402 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator); 403 EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator); 404 EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator); 405 EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator); 406 EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser); 407 EXPECT_EQ(Tokens[31]->FakeRParens, 1u); 408 EXPECT_TRUE(Tokens[31]->ClosesRequiresClause); 409 410 Tokens = 411 annotate("template<typename T>\n" 412 "requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n" 413 "struct Foo;"); 414 ASSERT_EQ(Tokens.size(), 38u) << Tokens; 415 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause); 416 EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown); 417 EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u); 418 EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator); 419 EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator); 420 EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator); 421 EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator); 422 EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser); 423 EXPECT_EQ(Tokens[32]->FakeRParens, 1u); 424 EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown); 425 EXPECT_TRUE(Tokens[33]->ClosesRequiresClause); 426 427 Tokens = annotate("template <typename T>\n" 428 "void foo(T) noexcept requires Bar<T>;"); 429 ASSERT_EQ(Tokens.size(), 18u) << Tokens; 430 EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause); 431 432 Tokens = annotate("template <typename T>\n" 433 "struct S {\n" 434 " void foo() const requires Bar<T>;\n" 435 " void bar() const & requires Baz<T>;\n" 436 " void bar() && requires Baz2<T>;\n" 437 " void baz() const & noexcept requires Baz<T>;\n" 438 " void baz() && noexcept requires Baz2<T>;\n" 439 "};\n" 440 "\n" 441 "void S::bar() const & requires Baz<T> { }"); 442 ASSERT_EQ(Tokens.size(), 85u) << Tokens; 443 EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause); 444 EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause); 445 EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause); 446 EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause); 447 EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause); 448 EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause); 449 450 Tokens = annotate("void Class::member() && requires(Constant) {}"); 451 ASSERT_EQ(Tokens.size(), 14u) << Tokens; 452 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause); 453 454 Tokens = annotate("void Class::member() && requires(Constant<T>) {}"); 455 ASSERT_EQ(Tokens.size(), 17u) << Tokens; 456 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause); 457 458 Tokens = 459 annotate("void Class::member() && requires(Namespace::Constant<T>) {}"); 460 ASSERT_EQ(Tokens.size(), 19u) << Tokens; 461 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause); 462 463 Tokens = annotate("void Class::member() && requires(typename " 464 "Namespace::Outer<T>::Inner::Constant) {}"); 465 ASSERT_EQ(Tokens.size(), 24u) << Tokens; 466 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause); 467 468 Tokens = annotate("struct [[nodiscard]] zero_t {\n" 469 " template<class T>\n" 470 " requires requires { number_zero_v<T>; }\n" 471 " [[nodiscard]] constexpr operator T() const { " 472 "return number_zero_v<T>; }\n" 473 "};"); 474 ASSERT_EQ(Tokens.size(), 44u); 475 EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause); 476 EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression); 477 EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace); 478 EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown); 479 EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]); 480 EXPECT_TRUE(Tokens[21]->ClosesRequiresClause); 481 482 Tokens = 483 annotate("template <class A, class B> concept C =" 484 "std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;"); 485 ASSERT_EQ(Tokens.size(), 31u) << Tokens; 486 EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown); 487 EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener); 488 EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener); 489 EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser); 490 EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener); 491 EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser); 492 EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser); 493 494 Tokens = annotate("auto bar() -> int requires(is_integral_v<T>) {}"); 495 ASSERT_EQ(Tokens.size(), 16u) << Tokens; 496 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause); 497 498 Tokens = annotate("auto bar() -> void requires(is_integral_v<T>) {}"); 499 ASSERT_EQ(Tokens.size(), 16u) << Tokens; 500 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause); 501 502 Tokens = annotate("auto bar() -> MyType requires(is_integral_v<T>) {}"); 503 ASSERT_EQ(Tokens.size(), 16u) << Tokens; 504 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause); 505 506 Tokens = 507 annotate("auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}"); 508 ASSERT_EQ(Tokens.size(), 16u) << Tokens; 509 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause); 510 511 Tokens = 512 annotate("auto bar() -> qualified::type requires(is_integral_v<T>) {}"); 513 ASSERT_EQ(Tokens.size(), 18u) << Tokens; 514 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause); 515 516 Tokens = 517 annotate("auto bar() -> Template<type> requires(is_integral_v<T>) {}"); 518 ASSERT_EQ(Tokens.size(), 19u) << Tokens; 519 EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause); 520 } 521 522 TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) { 523 auto Tokens = annotate("bool b = requires(int i) { i + 5; };"); 524 ASSERT_EQ(Tokens.size(), 16u) << Tokens; 525 EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression); 526 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen); 527 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace); 528 529 Tokens = annotate("if (requires(int i) { i + 5; }) return;"); 530 ASSERT_EQ(Tokens.size(), 17u) << Tokens; 531 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 532 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 533 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace); 534 535 Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;"); 536 ASSERT_EQ(Tokens.size(), 21u) << Tokens; 537 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression); 538 EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen); 539 EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace); 540 541 Tokens = annotate("foo(requires(const T t) {});"); 542 ASSERT_EQ(Tokens.size(), 13u) << Tokens; 543 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 544 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 545 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace); 546 547 Tokens = annotate("foo(requires(const int t) {});"); 548 ASSERT_EQ(Tokens.size(), 13u) << Tokens; 549 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 550 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 551 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace); 552 553 Tokens = annotate("foo(requires(const T t) {});"); 554 ASSERT_EQ(Tokens.size(), 13u) << Tokens; 555 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 556 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 557 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace); 558 559 Tokens = annotate("foo(requires(int const* volatile t) {});"); 560 ASSERT_EQ(Tokens.size(), 15u) << Tokens; 561 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 562 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 563 EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace); 564 565 Tokens = annotate("foo(requires(T const* volatile t) {});"); 566 ASSERT_EQ(Tokens.size(), 15u) << Tokens; 567 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 568 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 569 EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace); 570 571 Tokens = 572 annotate("foo(requires(const typename Outer<T>::Inner * const t) {});"); 573 ASSERT_EQ(Tokens.size(), 21u) << Tokens; 574 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression); 575 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen); 576 EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace); 577 578 Tokens = annotate("template <typename T>\n" 579 "concept C = requires(T T) {\n" 580 " requires Bar<T> && Foo<T>;\n" 581 "};"); 582 ASSERT_EQ(Tokens.size(), 28u) << Tokens; 583 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression); 584 EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen); 585 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace); 586 EXPECT_TOKEN(Tokens[14], tok::kw_requires, 587 TT_RequiresClauseInARequiresExpression); 588 589 Tokens = annotate("template <typename T>\n" 590 "concept C = requires(T T) {\n" 591 " { t.func() } -> std::same_as<int>;" 592 " requires Bar<T> && Foo<T>;\n" 593 "};"); 594 ASSERT_EQ(Tokens.size(), 43u) << Tokens; 595 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression); 596 EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen); 597 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace); 598 EXPECT_TOKEN(Tokens[29], tok::kw_requires, 599 TT_RequiresClauseInARequiresExpression); 600 601 // Invalid Code, but we don't want to crash. See http://llvm.org/PR54350. 602 Tokens = annotate("bool r10 = requires (struct new_struct { int x; } s) { " 603 "requires true; };"); 604 ASSERT_EQ(Tokens.size(), 21u) << Tokens; 605 EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression); 606 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen); 607 EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace); 608 } 609 610 TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) { 611 const char *BaseCode = nullptr; 612 const char *ConstrainedCode = nullptr; 613 auto BaseTokenCount = 0u; 614 auto RequiresTokenCount = 0u; 615 auto PrefixTokenCount = 0u; 616 617 auto TestRequires = [&](int Line) { 618 const auto BaseTokens = annotate(BaseCode); 619 const auto ConstrainedTokens = annotate(ConstrainedCode); 620 621 #define LINE " (Line " << Line << ')' 622 623 ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE; 624 ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount) 625 << LINE; 626 627 for (auto I = 0u; I < BaseTokenCount; ++I) { 628 EXPECT_EQ( 629 *BaseTokens[I], 630 *ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount]) 631 << I << LINE; 632 } 633 634 #undef LINE 635 }; 636 637 BaseCode = "template<typename T>\n" 638 "T Pi = 3.14;"; 639 ConstrainedCode = "template<typename T>\n" 640 " requires Foo<T>\n" 641 "T Pi = 3.14;"; 642 BaseTokenCount = 11; 643 RequiresTokenCount = 5; 644 PrefixTokenCount = 5; 645 TestRequires(__LINE__); 646 647 BaseCode = "template<typename T>\n" 648 "struct Bar;"; 649 ConstrainedCode = "template<typename T>\n" 650 " requires Foo<T>\n" 651 "struct Bar;"; 652 BaseTokenCount = 9; 653 TestRequires(__LINE__); 654 655 BaseCode = "template<typename T>\n" 656 "struct Bar {\n" 657 " T foo();\n" 658 " T bar();\n" 659 "};"; 660 ConstrainedCode = "template<typename T>\n" 661 " requires Foo<T>\n" 662 "struct Bar {\n" 663 " T foo();\n" 664 " T bar();\n" 665 "};"; 666 BaseTokenCount = 21; 667 TestRequires(__LINE__); 668 669 BaseCode = "template<typename T>\n" 670 "Bar(T) -> Bar<T>;"; 671 ConstrainedCode = "template<typename T>\n" 672 " requires Foo<T>\n" 673 "Bar(T) -> Bar<T>;"; 674 BaseTokenCount = 16; 675 TestRequires(__LINE__); 676 677 BaseCode = "template<typename T>\n" 678 "T foo();"; 679 ConstrainedCode = "template<typename T>\n" 680 " requires Foo<T>\n" 681 "T foo();"; 682 BaseTokenCount = 11; 683 TestRequires(__LINE__); 684 685 BaseCode = "template<typename T>\n" 686 "T foo() {\n" 687 " auto bar = baz();\n" 688 " return bar + T{};\n" 689 "}"; 690 ConstrainedCode = "template<typename T>\n" 691 " requires Foo<T>\n" 692 "T foo() {\n" 693 " auto bar = baz();\n" 694 " return bar + T{};\n" 695 "}"; 696 BaseTokenCount = 26; 697 TestRequires(__LINE__); 698 699 BaseCode = "template<typename T>\n" 700 "T foo();"; 701 ConstrainedCode = "template<typename T>\n" 702 "T foo() requires Foo<T>;"; 703 BaseTokenCount = 11; 704 PrefixTokenCount = 9; 705 TestRequires(__LINE__); 706 707 BaseCode = "template<typename T>\n" 708 "T foo() {\n" 709 " auto bar = baz();\n" 710 " return bar + T{};\n" 711 "}"; 712 ConstrainedCode = "template<typename T>\n" 713 "T foo() requires Foo<T> {\n" 714 " auto bar = baz();\n" 715 " return bar + T{};\n" 716 "}"; 717 BaseTokenCount = 26; 718 TestRequires(__LINE__); 719 720 BaseCode = "template<typename T>\n" 721 "Bar(T) -> Bar<typename T::I>;"; 722 ConstrainedCode = "template<typename T>\n" 723 " requires requires(T &&t) {\n" 724 " typename T::I;\n" 725 " }\n" 726 "Bar(T) -> Bar<typename T::I>;"; 727 BaseTokenCount = 19; 728 RequiresTokenCount = 14; 729 PrefixTokenCount = 5; 730 TestRequires(__LINE__); 731 732 BaseCode = "struct [[nodiscard]] zero_t {\n" 733 " template<class T>\n" 734 " [[nodiscard]] constexpr operator T() const { return v<T>; }\n" 735 "};"; 736 ConstrainedCode = 737 "struct [[nodiscard]] zero_t {\n" 738 " template<class T>\n" 739 " requires requires { v<T>; }\n" 740 " [[nodiscard]] constexpr operator T() const { return v<T>; }\n" 741 "};"; 742 BaseTokenCount = 35; 743 RequiresTokenCount = 9; 744 PrefixTokenCount = 13; 745 TestRequires(__LINE__); 746 747 BaseCode = "constexpr Foo(Foo const &other)\n" 748 " : value{other.value} {\n" 749 " do_magic();\n" 750 " do_more_magic();\n" 751 "}"; 752 ConstrainedCode = "constexpr Foo(Foo const &other)\n" 753 " requires std::is_copy_constructible<T>\n" 754 " : value{other.value} {\n" 755 " do_magic();\n" 756 " do_more_magic();\n" 757 "}"; 758 BaseTokenCount = 26; 759 RequiresTokenCount = 7; 760 PrefixTokenCount = 8; 761 TestRequires(__LINE__); 762 763 BaseCode = "constexpr Foo(Foo const &other)\n" 764 " : value{other.value} {\n" 765 " do_magic();\n" 766 " do_more_magic();\n" 767 "}"; 768 ConstrainedCode = "constexpr Foo(Foo const &other)\n" 769 " requires (std::is_copy_constructible<T>)\n" 770 " : value{other.value} {\n" 771 " do_magic();\n" 772 " do_more_magic();\n" 773 "}"; 774 RequiresTokenCount = 9; 775 TestRequires(__LINE__); 776 } 777 778 TEST_F(TokenAnnotatorTest, UnderstandsAsm) { 779 auto Tokens = annotate("__asm{\n" 780 "a:\n" 781 "};"); 782 ASSERT_EQ(Tokens.size(), 7u) << Tokens; 783 EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); 784 EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace); 785 EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace); 786 } 787 788 TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) { 789 auto Tokens = annotate("int (^)() = ^ ()\n" 790 " external_source_symbol() { //\n" 791 " return 1;\n" 792 "};"); 793 ASSERT_EQ(Tokens.size(), 21u) << Tokens; 794 EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen); 795 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace); 796 797 Tokens = annotate("int *p = ^int*(){ //\n" 798 " return nullptr;\n" 799 "}();"); 800 ASSERT_EQ(Tokens.size(), 19u) << Tokens; 801 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace); 802 } 803 804 TEST_F(TokenAnnotatorTest, UnderstandsLambdas) { 805 auto Tokens = annotate("[]() constexpr {}"); 806 ASSERT_EQ(Tokens.size(), 8u) << Tokens; 807 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 808 EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace); 809 810 Tokens = annotate("[]() -> auto {}"); 811 ASSERT_EQ(Tokens.size(), 9u) << Tokens; 812 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 813 EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow); 814 EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace); 815 816 Tokens = annotate("[]() -> auto & {}"); 817 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 818 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 819 EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow); 820 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace); 821 822 Tokens = annotate("[]() -> auto * {}"); 823 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 824 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 825 EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow); 826 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace); 827 828 Tokens = annotate("[] {}"); 829 ASSERT_EQ(Tokens.size(), 5u) << Tokens; 830 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 831 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace); 832 833 Tokens = annotate("[] noexcept {}"); 834 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 835 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 836 EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace); 837 838 Tokens = annotate("[] -> auto {}"); 839 ASSERT_EQ(Tokens.size(), 7u) << Tokens; 840 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 841 EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow); 842 EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace); 843 844 Tokens = annotate("[] <typename T> () {}"); 845 ASSERT_EQ(Tokens.size(), 11u) << Tokens; 846 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 847 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 848 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace); 849 850 Tokens = annotate("[] <typename T> {}"); 851 ASSERT_EQ(Tokens.size(), 9u) << Tokens; 852 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 853 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 854 EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace); 855 856 Tokens = annotate("[] <typename... T> () {}"); 857 ASSERT_EQ(Tokens.size(), 12u) << Tokens; 858 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 859 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 860 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace); 861 862 Tokens = annotate("[] <typename... T> {}"); 863 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 864 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 865 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 866 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace); 867 868 Tokens = annotate("[] <int... T> () {}"); 869 ASSERT_EQ(Tokens.size(), 12u) << Tokens; 870 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 871 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 872 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace); 873 874 Tokens = annotate("[] <int... T> {}"); 875 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 876 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 877 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 878 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace); 879 880 Tokens = annotate("[] <Foo... T> () {}"); 881 ASSERT_EQ(Tokens.size(), 12u) << Tokens; 882 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 883 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 884 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace); 885 886 Tokens = annotate("[] <Foo... T> {}"); 887 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 888 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare); 889 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener); 890 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace); 891 } 892 893 TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) { 894 auto Tokens = annotate("template <typename T>\n" 895 "DEPRECATED(\"Use NewClass::NewFunction instead.\")\n" 896 "string OldFunction(const string ¶meter) {}"); 897 ASSERT_EQ(Tokens.size(), 20u) << Tokens; 898 EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen); 899 900 Tokens = annotate("template <typename T>\n" 901 "A(T) noexcept;"); 902 ASSERT_EQ(Tokens.size(), 12u) << Tokens; 903 EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown); 904 } 905 906 TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) { 907 auto Annotate = [this](llvm::StringRef Code) { 908 return annotate(Code, getLLVMStyle(FormatStyle::LK_Verilog)); 909 }; 910 // Test that unary operators get labeled as such and that operators like '++' 911 // don't get split. 912 tok::TokenKind Unary[] = {tok::plus, tok::minus, tok::exclaim, 913 tok::tilde, tok::amp, tok::pipe, 914 tok::caret, tok::plusplus, tok::minusminus}; 915 for (auto Kind : Unary) { 916 auto Tokens = 917 Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;"); 918 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 919 EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator); 920 } 921 // Operators formed by joining two operators like '^~'. For some of these 922 // joined operators, we don't have a separate type, so we only test for their 923 // precedence. 924 std::pair<prec::Level, std::string> JoinedBinary[] = { 925 {prec::Comma, "<->"}, {prec::Assignment, "+="}, 926 {prec::Assignment, "-="}, {prec::Assignment, "*="}, 927 {prec::Assignment, "/="}, {prec::Assignment, "%="}, 928 {prec::Assignment, "&="}, {prec::Assignment, "^="}, 929 {prec::Assignment, "<<="}, {prec::Assignment, ">>="}, 930 {prec::Assignment, "<<<="}, {prec::Assignment, ">>>="}, 931 {prec::LogicalOr, "||"}, {prec::LogicalAnd, "&&"}, 932 {prec::Equality, "=="}, {prec::Equality, "!="}, 933 {prec::Equality, "==="}, {prec::Equality, "!=="}, 934 {prec::Equality, "==?"}, {prec::Equality, "!=?"}, 935 {prec::ExclusiveOr, "~^"}, {prec::ExclusiveOr, "^~"}, 936 }; 937 for (auto Operator : JoinedBinary) { 938 auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;"); 939 ASSERT_EQ(Tokens.size(), 7u) << Tokens; 940 EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator); 941 EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first); 942 } 943 // '~^' and '^~' can be unary as well as binary operators. 944 auto Tokens = Annotate("x = ~^x;"); 945 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 946 EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator); 947 Tokens = Annotate("x = ^~x;"); 948 ASSERT_EQ(Tokens.size(), 6u) << Tokens; 949 EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator); 950 // The unary operators '~&' and '~|' can only be unary operators. The current 951 // implementation treats each of them as separate unary '~' and '&' or '|' 952 // operators, which is enough for formatting purposes. In FormatTestVerilog, 953 // there is a test that there is no space in between. And even if a new line 954 // is inserted between the '~' and '|', the semantic meaning is the same as 955 // the joined operator, so the CanBreakBefore property doesn't need to be 956 // false for the second operator. 957 Tokens = Annotate("x = ~&x;"); 958 ASSERT_EQ(Tokens.size(), 7u) << Tokens; 959 EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator); 960 EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator); 961 Tokens = Annotate("x = ~|x;"); 962 ASSERT_EQ(Tokens.size(), 7u) << Tokens; 963 EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator); 964 EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator); 965 // Test for block label colons. 966 Tokens = Annotate("begin : x\n" 967 "end : x"); 968 ASSERT_EQ(Tokens.size(), 7u); 969 EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon); 970 EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon); 971 // Test that the dimension colon is annotated correctly. 972 Tokens = Annotate("var [1 : 0] x;"); 973 ASSERT_EQ(Tokens.size(), 9u) << Tokens; 974 EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon); 975 Tokens = Annotate("extern function [1 : 0] x;"); 976 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 977 EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon); 978 // Test case labels and ternary operators. 979 Tokens = Annotate("case (x)\n" 980 " x:\n" 981 " x;\n" 982 "endcase\n"); 983 ASSERT_EQ(Tokens.size(), 10u) << Tokens; 984 EXPECT_TOKEN(Tokens[5], tok::colon, TT_GotoLabelColon); 985 Tokens = Annotate("case (x)\n" 986 " x ? x : x:\n" 987 " x;\n" 988 "endcase\n"); 989 ASSERT_EQ(Tokens.size(), 14u) << Tokens; 990 EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr); 991 EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr); 992 EXPECT_TOKEN(Tokens[9], tok::colon, TT_GotoLabelColon); 993 } 994 995 } // namespace 996 } // namespace format 997 } // namespace clang 998