xref: /llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp (revision e5b05a51b8151cc7788bbdea4d491e5ccfceedea)
1 //===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Format/Format.h"
10 
11 #include "FormatTestUtils.h"
12 #include "TestLexer.h"
13 #include "gtest/gtest.h"
14 
15 namespace clang {
16 namespace format {
17 
18 // Not really the equality, but everything we need.
19 static bool operator==(const FormatToken &LHS,
20                        const FormatToken &RHS) noexcept {
21   return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22          LHS.getType() == RHS.getType();
23 }
24 
25 namespace {
26 
27 class TokenAnnotatorTest : public testing::Test {
28 protected:
29   TokenList annotate(StringRef Code,
30                      const FormatStyle &Style = getLLVMStyle()) {
31     return TestLexer(Allocator, Buffers, Style).annotate(Code);
32   }
33   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35 };
36 
37 #define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \
38   EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39 #define EXPECT_TOKEN_TYPE(FormatTok, Type)                                     \
40   EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41 #define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec)                               \
42   EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)
43 #define EXPECT_BRACE_KIND(FormatTok, Kind)                                     \
44   EXPECT_EQ(FormatTok->getBlockKind(), Kind) << *(FormatTok)
45 #define EXPECT_SPLIT_PENALTY(FormatTok, Penalty)                               \
46   EXPECT_EQ(FormatTok->SplitPenalty, Penalty) << *(FormatTok)
47 #define EXPECT_TOKEN(FormatTok, Kind, Type)                                    \
48   do {                                                                         \
49     EXPECT_TOKEN_KIND(FormatTok, Kind);                                        \
50     EXPECT_TOKEN_TYPE(FormatTok, Type);                                        \
51   } while (false)
52 
53 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
54   auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};");
55   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
56   EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
57   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
58   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
59   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
60   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
61 
62   Tokens = annotate("auto x = [](const decltype(x) *ptr) {};");
63   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
64   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
65   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
66 
67   Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}");
68   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
69   EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
70   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
71   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
72   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
73   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
74 
75   Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}");
76   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
77   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
78   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
79 
80   Tokens = annotate("void f() {\n"
81                     "  while (p < a && *p == 'a')\n"
82                     "    p++;\n"
83                     "}");
84   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
85   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
86   EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
87 
88   Tokens = annotate("case *x:");
89   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
90   EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);
91   Tokens = annotate("case &x:");
92   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
93   EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);
94 
95   Tokens = annotate("bool b = 3 == int{3} && true;");
96   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
97   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
98 
99   Tokens = annotate("struct {\n"
100                     "} *ptr;");
101   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
102   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
103   Tokens = annotate("union {\n"
104                     "} *ptr;");
105   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
106   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
107   Tokens = annotate("class {\n"
108                     "} *ptr;");
109   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
110   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
111 
112   Tokens = annotate("struct {\n"
113                     "} &&ptr = {};");
114   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
115   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
116   Tokens = annotate("union {\n"
117                     "} &&ptr = {};");
118   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
119   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
120   Tokens = annotate("class {\n"
121                     "} &&ptr = {};");
122   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
123   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
124   Tokens = annotate("int i = int{42} * 2;");
125   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
126   EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);
127 
128   Tokens = annotate("delete[] *ptr;");
129   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
130   EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
131   Tokens = annotate("delete[] **ptr;");
132   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
133   EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
134   EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);
135   Tokens = annotate("delete[] *(ptr);");
136   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
137   EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
138 
139   Tokens = annotate("void f() { void (*fnptr)(char* foo); }");
140   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
141   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
142   // FIXME: The star of a function pointer probably makes more sense as
143   // TT_PointerOrReference.
144   EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
145   EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
146 
147   Tokens = annotate("void f() { void (*fnptr)(t* foo); }");
148   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
149   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
150   EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
151   EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
152 
153   Tokens = annotate("int f3() { return sizeof(Foo&); }");
154   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
155   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
156 
157   Tokens = annotate("int f4() { return sizeof(Foo&&); }");
158   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
159   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
160 
161   Tokens = annotate("void f5() { int f6(Foo&, Bar&); }");
162   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
163   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
164   EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
165 
166   Tokens = annotate("void f7() { int f8(Foo&&, Bar&&); }");
167   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
168   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
169   EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
170 
171   Tokens = annotate("Type1 &val1 = val2;");
172   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
173   EXPECT_TOKEN(Tokens[1], tok::amp, TT_PointerOrReference);
174 
175   Tokens = annotate("Type1 *val1 = &val2;");
176   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
177   EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
178   EXPECT_TOKEN(Tokens[4], tok::amp, TT_UnaryOperator);
179 
180   Tokens = annotate("val1 & val2;");
181   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
182   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
183 
184   Tokens = annotate("val1 & val2.member;");
185   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
186   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
187 
188   Tokens = annotate("val1 & val2.*member;");
189   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
190   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
191 
192   Tokens = annotate("val1.*member & val2;");
193   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
194   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
195 
196   Tokens = annotate("val1 & val2->*member;");
197   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
198   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
199 
200   Tokens = annotate("val1->member & val2;");
201   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
202   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
203 
204   Tokens = annotate("val1 & val2 & val3;");
205   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
206   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
207   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
208 
209   Tokens = annotate("val1 & val2 // comment\n"
210                     "     & val3;");
211   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
212   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
213   EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
214 
215   Tokens =
216       annotate("val1 & val2.member & val3.member() & val4 & val5->member;");
217   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
218   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
219   EXPECT_TOKEN(Tokens[5], tok::amp, TT_BinaryOperator);
220   EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
221   EXPECT_TOKEN(Tokens[13], tok::amp, TT_BinaryOperator);
222 
223   Tokens = annotate("class c {\n"
224                     "  void func(type &a) { a & member; }\n"
225                     "  anotherType &member;\n"
226                     "}");
227   ASSERT_EQ(Tokens.size(), 22u) << Tokens;
228   EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
229   EXPECT_TOKEN(Tokens[12], tok::amp, TT_BinaryOperator);
230   EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);
231 
232   Tokens = annotate("struct S {\n"
233                     "  auto Mem = C & D;\n"
234                     "}");
235   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
236   EXPECT_TOKEN(Tokens[7], tok::amp, TT_BinaryOperator);
237 
238   Tokens =
239       annotate("template <typename T> void swap() noexcept(Bar<T> && Foo<T>);");
240   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
241   EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
242 
243   Tokens = annotate("template <typename T> struct S {\n"
244                     "  explicit(Bar<T> && Foo<T>) S(const S &);\n"
245                     "};");
246   ASSERT_EQ(Tokens.size(), 30u) << Tokens;
247   EXPECT_TOKEN(Tokens[14], tok::ampamp, TT_BinaryOperator);
248 
249   Tokens = annotate("template <bool B = C && D> struct S {};");
250   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
251   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
252 
253   Tokens = annotate("template <typename T, bool B = C && D> struct S {};");
254   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
255   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
256 
257   Tokens = annotate("template <typename T, typename U = T&&> struct S {};");
258   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
259   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
260 
261   Tokens = annotate("template <typename T = int (*)(int)> struct S {};");
262   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
263   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
264   EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);
265 
266   Tokens = annotate("Foo<A && B> a = {};");
267   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
268   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
269 
270   Tokens = annotate("Foo<A &&> a = {};");
271   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
272   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
273 
274   Tokens = annotate("template <typename T>\n"
275                     "enable_if_t<is_integral_v<T>, bool> // comment\n"
276                     "operator~(T &a);");
277   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
278   EXPECT_TOKEN(Tokens[19], tok::amp, TT_PointerOrReference);
279 
280   Tokens = annotate("template <enable_if_t<foo && !bar>* = nullptr> void f();");
281   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
282   EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_BinaryOperator);
283 
284   Tokens =
285       annotate("auto foo() noexcept(noexcept(bar()) && "
286                "trait<std::decay_t<decltype(bar())>> && noexcept(baz())) {}");
287   ASSERT_EQ(Tokens.size(), 38u) << Tokens;
288   EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
289   EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
290 
291   Tokens = annotate("foo = *i < *j && *j > *k;");
292   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
293   EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
294   EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
295   EXPECT_TOKEN(Tokens[10], tok::greater, TT_BinaryOperator);
296 
297   Tokens = annotate("if (Foo *foo; bar)");
298   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
299   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
300 
301   Tokens = annotate("if (Foo **foo(); bar)");
302   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
303   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
304   EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
305 
306   Tokens = annotate("if (Foo *&foo{a}; bar)");
307   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
308   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
309   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
310 
311   Tokens = annotate("if (Foo *&foo{a})");
312   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
313   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
314   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
315 
316   FormatStyle Style = getLLVMStyle();
317   Style.TypeNames.push_back("MYI");
318   Tokens = annotate("if (MYI *p{nullptr})", Style);
319   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
320   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
321   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
322 
323   Style.TypeNames.push_back("Class");
324   Tokens = annotate("if (Class *obj {getObj()})", Style);
325   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
326   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
327   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
328 
329   Tokens = annotate("class Foo {\n"
330                     "  void operator<() {}\n"
331                     "  Foo &f;\n"
332                     "};");
333   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
334   EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
335   EXPECT_TOKEN(Tokens[5], tok::less, TT_OverloadedOperator);
336   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
337   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
338   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
339 
340   Tokens = annotate("if (new && num) {\n"
341                     "  new = 1;\n"
342                     "}\n"
343                     "if (!delete && num) {\n"
344                     "  delete = 1;\n"
345                     "}");
346   ASSERT_EQ(Tokens.size(), 26u) << Tokens;
347   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
348   EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
349 
350   Tokens = annotate("#define FOO \\\n"
351                     "  void foo() { f(a * b); }");
352   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
353   EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
354 
355   Tokens = annotate("for (int i; Foo *&foo : foos)");
356   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
357   EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
358   EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
359   EXPECT_TOKEN(Tokens[9], tok::colon, TT_RangeBasedForLoopColon);
360 
361   Tokens = annotate("#define FOO auto Foo = [] { f(a * b); };");
362   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
363   EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
364 
365   Tokens = annotate("namespace {\n"
366                     "#define FOO(x) void foo(a##x *b);\n"
367                     "}");
368   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
369   EXPECT_TOKEN(Tokens[14], tok::star, TT_PointerOrReference);
370 
371   Tokens = annotate("Thingy kConfig = {\n"
372                     "    1,\n"
373                     "    (uint16_t)(kScale * height_pixels),\n"
374                     "};");
375   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
376   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_CastRParen);
377   EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
378 }
379 
380 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
381   auto Tokens = annotate("x - 0");
382   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
383   EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);
384   Tokens = annotate("0 + 0");
385   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
386   EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);
387   Tokens = annotate("x + +0");
388   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
389   EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);
390   Tokens = annotate("x ? -0 : +0");
391   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
392   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
393   EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);
394   Tokens = annotate("(-0)");
395   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
396   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
397   Tokens = annotate("0, -0");
398   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
399   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
400   Tokens = annotate("for (; -1;) {\n}");
401   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
402   EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
403   Tokens = annotate("x = -1;");
404   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
405   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
406   Tokens = annotate("x[-1]");
407   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
408   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
409   Tokens = annotate("x = {-1};");
410   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
411   EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
412   Tokens = annotate("case -x:");
413   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
414   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
415   Tokens = annotate("co_await -x;");
416   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
417   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
418   Tokens = annotate("co_return -x;");
419   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
420   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
421   Tokens = annotate("co_yield -x;");
422   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
423   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
424   Tokens = annotate("delete -x;");
425   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
426   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
427   Tokens = annotate("return -x;");
428   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
429   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
430   Tokens = annotate("throw -x;");
431   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
432   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
433   Tokens = annotate("sizeof -x");
434   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
435   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
436   Tokens = annotate("co_await +x;");
437   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
438   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
439   Tokens = annotate("co_return +x;");
440   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
441   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
442   Tokens = annotate("co_yield +x;");
443   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
444   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
445   Tokens = annotate("delete +x;");
446   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
447   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
448   Tokens = annotate("return +x;");
449   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
450   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
451   Tokens = annotate("throw +x;");
452   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
453   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
454   Tokens = annotate("sizeof +x");
455   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
456   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
457   Tokens = annotate("(int)-x");
458   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
459   EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
460   Tokens = annotate("(-x)");
461   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
462   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
463   Tokens = annotate("!+x");
464   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
465   EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);
466   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
467 }
468 
469 TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
470   auto Tokens = annotate("class C {};");
471   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
472   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
473   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
474 
475   Tokens = annotate("const class C {} c;");
476   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
477   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
478   EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_ClassRBrace);
479 
480   Tokens = annotate("const class {} c;");
481   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
482   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
483   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
484 
485   Tokens = annotate("class [[deprecated(\"\")]] C { int i; };");
486   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
487   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
488   EXPECT_TOKEN(Tokens[14], tok::r_brace, TT_ClassRBrace);
489 }
490 
491 TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
492   auto Tokens = annotate("struct S {};");
493   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
494   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
495   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_StructRBrace);
496 
497   Tokens = annotate("struct macro(a) S {};");
498   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
499   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
500   EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_StructRBrace);
501 
502   Tokens = annotate("struct EXPORT_MACRO [[nodiscard]] C { int i; };");
503   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
504   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
505   EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_StructRBrace);
506 
507   Tokens = annotate("struct [[deprecated]] [[nodiscard]] C { int i; };");
508   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
509   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
510   EXPECT_TOKEN(Tokens[16], tok::r_brace, TT_StructRBrace);
511 
512   Tokens = annotate("struct macro(a) S {\n"
513                     "  void f(T &t);\n"
514                     "};");
515   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
516   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
517   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
518   EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
519 
520   Tokens = annotate("template <typename T> struct S<const T[N]> {};");
521   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
522   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
523   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
524   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
525   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
526   EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
527 
528   Tokens = annotate("template <typename T> struct S<T const[N]> {};");
529   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
530   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
531   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
532   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
533   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
534   EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
535 
536   Tokens = annotate("template <typename T, unsigned n> struct S<T const[n]> {\n"
537                     "  void f(T const (&a)[n]);\n"
538                     "};");
539   ASSERT_EQ(Tokens.size(), 35u) << Tokens;
540   EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);
541   EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);
542   EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
543   EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_StructLBrace);
544   EXPECT_TOKEN(Tokens[23], tok::l_paren, TT_FunctionTypeLParen);
545   EXPECT_TOKEN(Tokens[24], tok::amp, TT_UnaryOperator);
546   EXPECT_TOKEN(Tokens[27], tok::l_square, TT_ArraySubscriptLSquare);
547   EXPECT_TOKEN(Tokens[32], tok::r_brace, TT_StructRBrace);
548 
549   Tokens = annotate("template <typename T, enum E e> struct S {};");
550   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
551   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_StructLBrace);
552 
553   auto Style = getLLVMStyle();
554   Style.AttributeMacros.push_back("EXPORT");
555   Tokens = annotate("struct EXPORT StructName {};", Style);
556   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
557   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_AttributeMacro);
558   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
559   EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
560 }
561 
562 TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
563   auto Tokens = annotate("union U {};");
564   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
565   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
566   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_UnionRBrace);
567 
568   Tokens = annotate("union U { void f() { return; } };");
569   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
570   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
571   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
572   EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_UnionRBrace);
573 }
574 
575 TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
576   auto Tokens = annotate("enum E {};");
577   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
578   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
579   EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_EnumRBrace);
580 }
581 
582 TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
583   auto Tokens = annotate("auto operator<=>(const T &) const & = default;");
584   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
585   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
586 
587   Tokens = annotate("template <typename T> void F(T) && = delete;");
588   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
589   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
590 }
591 
592 TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
593   auto Tokens =
594       annotate("inline bool var = is_integral_v<int> && is_signed_v<int>;");
595   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
596   EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
597 }
598 
599 TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
600   auto Tokens =
601       annotate("template <typename T> "
602                "inline bool var = is_integral_v<int> && is_signed_v<int>;");
603   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
604   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
605 }
606 
607 TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {
608   auto Tokens =
609       annotate("#define FOO(typeName) \\\n"
610                "  { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");
611   ASSERT_EQ(Tokens.size(), 27u) << Tokens;
612   EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
613   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
614   EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);
615   EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);
616 }
617 
618 TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {
619   auto Tokens = annotate("if (std::tuple_size_v<T> > 0)");
620   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
621   EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
622   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
623   EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
624 }
625 
626 TEST_F(TokenAnnotatorTest, UnderstandsTernaryInTemplate) {
627   // IsExpression = false
628   auto Tokens = annotate("foo<true ? 1 : 2>();");
629   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
630   EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
631   EXPECT_TOKEN(Tokens[3], tok::question, TT_ConditionalExpr);
632   EXPECT_TOKEN(Tokens[5], tok::colon, TT_ConditionalExpr);
633   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
634 
635   // IsExpression = true
636 
637   Tokens = annotate("return foo<true ? 1 : 2>();");
638   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
639   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
640   EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);
641   EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);
642   EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
643 
644   Tokens = annotate("return foo<true ? 1 : 2>{};");
645   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
646   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
647   EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);
648   EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);
649   EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
650 }
651 
652 TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {
653   auto Tokens = annotate("return a < b && c > d;");
654   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
655   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
656   EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
657 
658   Tokens = annotate("a < 0 ? b : a > 0 ? c : d;");
659   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
660   EXPECT_TOKEN(Tokens[1], tok::less, TT_BinaryOperator);
661   EXPECT_TOKEN(Tokens[7], tok::greater, TT_BinaryOperator);
662 
663   Tokens = annotate("return A < B ? true : A > B;");
664   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
665   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
666   EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
667 
668   Tokens = annotate("return A < B ? true : A > B ? false : false;");
669   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
670   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
671   EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
672 
673   Tokens = annotate("return checklower ? a < b : a > b;");
674   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
675   EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
676   EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
677 
678   Tokens = annotate("return A < B ^ A > B;");
679   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
680   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
681   EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
682 
683   Tokens = annotate("ratio{-1, 2} < ratio{-1, 3} == -1 / 3 > -1 / 2;");
684   ASSERT_EQ(Tokens.size(), 27u) << Tokens;
685   EXPECT_TOKEN(Tokens[7], tok::less, TT_BinaryOperator);
686   EXPECT_TOKEN(Tokens[20], tok::greater, TT_BinaryOperator);
687 }
688 
689 TEST_F(TokenAnnotatorTest, UnderstandsTemplateTemplateParameters) {
690   auto Tokens = annotate("template <template <typename...> typename X,\n"
691                          "          template <typename...> class Y,\n"
692                          "          typename... T>\n"
693                          "class A {};");
694   ASSERT_EQ(Tokens.size(), 28u) << Tokens;
695   EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
696   EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
697   EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
698   EXPECT_FALSE(Tokens[6]->ClosesTemplateDeclaration);
699   EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
700   EXPECT_TOKEN(Tokens[14], tok::greater, TT_TemplateCloser);
701   EXPECT_FALSE(Tokens[14]->ClosesTemplateDeclaration);
702   EXPECT_TOKEN(Tokens[21], tok::greater, TT_TemplateCloser);
703   EXPECT_TRUE(Tokens[21]->ClosesTemplateDeclaration);
704 }
705 
706 TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
707   FormatStyle Style = getLLVMStyle();
708   Style.WhitespaceSensitiveMacros.push_back("FOO");
709 
710   auto Tokens = annotate("FOO(1+2 )", Style);
711   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
712   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
713 
714   Tokens = annotate("FOO(a:b:c)", Style);
715   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
716   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
717 }
718 
719 TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
720   auto Tokens = annotate("delete (void *)p;");
721   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
722   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
723 
724   Tokens = annotate("delete[] (void *)p;");
725   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
726   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
727 
728   Tokens = annotate("delete[] /*comment*/ (void *)p;");
729   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
730   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
731 
732   Tokens = annotate("delete[/*comment*/] (void *)p;");
733   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
734   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
735 
736   Tokens = annotate("delete/*comment*/[] (void *)p;");
737   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
738   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
739 }
740 
741 TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
742   auto Tokens = annotate("(void)p;");
743   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
744   EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
745 
746   Tokens = annotate("(uint32_t)&&label;");
747   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
748   EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
749   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_UnaryOperator);
750   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
751 
752   Tokens = annotate("auto x = (Foo)p;");
753   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
754   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
755 
756   Tokens = annotate("(std::vector<int>)p;");
757   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
758   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
759 
760   Tokens = annotate("return (Foo)p;");
761   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
762   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
763 
764   Tokens = annotate("throw (Foo)p;");
765   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
766   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
767 
768   Tokens = annotate("#define FOO(x) (((uint64_t)(x) * BAR) / 100)");
769   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
770   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
771   EXPECT_TOKEN(Tokens[13], tok::r_paren, TT_Unknown);
772   EXPECT_TOKEN(Tokens[14], tok::star, TT_BinaryOperator);
773 
774   Tokens = annotate("#define foo(i) ((i) - bar)");
775   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
776   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
777   EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
778 
779   Tokens = annotate("return (Foo) & 10;");
780   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
781   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
782   EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
783 
784   Tokens = annotate("return (struct foo){};");
785   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
786   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
787 
788   Tokens = annotate("#define FOO(bar) foo((uint64_t)&bar)");
789   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
790   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
791   EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
792 
793   Tokens = annotate("#define FOO(bar) foo((Foo) & bar)");
794   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
795   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_Unknown);
796   EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
797 
798   Tokens = annotate("func((void (*)())&a);");
799   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
800   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
801   EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);
802   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_CastRParen);
803   EXPECT_TOKEN(Tokens[10], tok::amp, TT_UnaryOperator);
804 
805   Tokens = annotate("int result = ((int)a) - b;");
806   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
807   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
808   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
809   EXPECT_TOKEN(Tokens[9], tok::minus, TT_BinaryOperator);
810 
811   Tokens = annotate("return (double)(foo(30)) - 15;");
812   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
813   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
814   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
815   EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
816 
817   Tokens = annotate("return (::Type)(1 + 2);");
818   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
819   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
820 
821   Tokens = annotate("return (Namespace::Class)(1 + 2);");
822   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
823   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
824 
825   Tokens = annotate("return (Foo (*)(void *, Bar, ...))&foo;");
826   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
827   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
828   EXPECT_TOKEN(Tokens[14], tok::r_paren, TT_CastRParen);
829   EXPECT_TOKEN(Tokens[15], tok::amp, TT_UnaryOperator);
830 
831   auto Style = getLLVMStyle();
832   Style.TypeNames.push_back("Foo");
833   Tokens = annotate("#define FOO(bar) foo((Foo)&bar)", Style);
834   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
835   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_TypeName);
836   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
837   EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
838 }
839 
840 TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {
841   auto Tokens = annotate("void f() throw(int);");
842   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
843   EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);
844 }
845 
846 TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
847   auto Tokens = annotate("void f() &;");
848   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
849   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
850 
851   Tokens = annotate("void operator=(T) &&;");
852   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
853   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
854 
855   Tokens = annotate("template <typename T> void f() &;");
856   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
857   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
858 
859   Tokens = annotate("template <typename T> void operator=(T) &;");
860   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
861   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
862 }
863 
864 TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
865   auto Tokens = annotate("x.operator+()");
866   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
867   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
868   EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);
869   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
870   Tokens = annotate("x.operator=()");
871   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
872   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
873   EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);
874   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
875   Tokens = annotate("x.operator+=()");
876   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
877   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
878   EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);
879   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
880   Tokens = annotate("x.operator,()");
881   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
882   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
883   EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);
884   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
885   Tokens = annotate("x.operator()()");
886   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
887   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
888   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);
889   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);
890   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
891   Tokens = annotate("x.operator[]()");
892   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
893   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
894   // EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);
895   // EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);
896   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
897   Tokens = annotate("x.operator\"\"_a()");
898   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
899   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
900   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
901   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
902   Tokens = annotate("x.operator\"\" _a()");
903   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
904   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
905   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
906   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
907   Tokens = annotate("x.operator\"\"if()");
908   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
909   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
910   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
911   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
912   Tokens = annotate("x.operator\"\"s()");
913   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
914   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
915   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
916   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
917   Tokens = annotate("x.operator\"\" s()");
918   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
919   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
920   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
921   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
922 
923   Tokens = annotate("int operator+(int);");
924   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
925   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
926   EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
927   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
928   Tokens = annotate("auto operator=(T&) {}");
929   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
930   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
931   EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);
932   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
933   Tokens = annotate("auto operator()() {}");
934   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
935   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
936   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);
937   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);
938   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
939 
940   Tokens = annotate("class Foo {\n"
941                     "  int operator+(a* b);\n"
942                     "}");
943   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
944   EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
945   EXPECT_TOKEN(Tokens[5], tok::plus, TT_OverloadedOperator);
946   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
947   EXPECT_TOKEN(Tokens[8], tok::star, TT_PointerOrReference);
948 
949   Tokens = annotate("class Foo {\n"
950                     "  int c = operator+(a * b);\n"
951                     "}");
952   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
953   EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);
954   EXPECT_TOKEN(Tokens[7], tok::plus, TT_OverloadedOperator);
955   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_OverloadedOperatorLParen);
956   EXPECT_TOKEN(Tokens[10], tok::star, TT_BinaryOperator);
957 
958   Tokens = annotate("void foo() {\n"
959                     "  operator+(a * b);\n"
960                     "}");
961   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
962   EXPECT_TOKEN(Tokens[5], tok::kw_operator, TT_Unknown);
963   EXPECT_TOKEN(Tokens[6], tok::plus, TT_OverloadedOperator);
964   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
965   EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
966 
967   Tokens = annotate("return operator+(a * b, c & d) + operator+(a && b && c);");
968   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
969   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_Unknown);
970   EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
971   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
972   EXPECT_TOKEN(Tokens[5], tok::star, TT_BinaryOperator);
973   EXPECT_TOKEN(Tokens[9], tok::amp, TT_BinaryOperator);
974   EXPECT_TOKEN(Tokens[13], tok::kw_operator, TT_Unknown);
975   EXPECT_TOKEN(Tokens[14], tok::plus, TT_OverloadedOperator);
976   EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_OverloadedOperatorLParen);
977   EXPECT_TOKEN(Tokens[17], tok::ampamp, TT_BinaryOperator);
978   EXPECT_TOKEN(Tokens[19], tok::ampamp, TT_BinaryOperator);
979 
980   Tokens = annotate("class Foo {\n"
981                     "  void foo() {\n"
982                     "    operator+(a * b);\n"
983                     "  }\n"
984                     "}");
985   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
986   EXPECT_TOKEN(Tokens[8], tok::kw_operator, TT_Unknown);
987   EXPECT_TOKEN(Tokens[9], tok::plus, TT_OverloadedOperator);
988   EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_OverloadedOperatorLParen);
989   EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
990 
991   Tokens = annotate("std::vector<Foo> operator()(Foo &foo);");
992   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
993   EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
994   EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
995   EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_FunctionDeclarationName);
996   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperator);
997   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_OverloadedOperator);
998   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_OverloadedOperatorLParen);
999   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
1000 
1001   Tokens = annotate("decltype(auto) operator()(T &x);");
1002   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1003   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
1004   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_TypeDeclarationParen);
1005   EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1006   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperator);
1007   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_OverloadedOperator);
1008   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1009   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
1010 }
1011 
1012 TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {
1013   struct {
1014     const char *Text;
1015     tok::TokenKind Kind;
1016   } Operators[] = {{"+", tok::plus},
1017                    {"-", tok::minus},
1018                    // FIXME:
1019                    // {"*", tok::star},
1020                    {"/", tok::slash},
1021                    {"%", tok::percent},
1022                    {"^", tok::caret},
1023                    // FIXME:
1024                    // {"&", tok::amp},
1025                    {"|", tok::pipe},
1026                    {"~", tok::tilde},
1027                    {"!", tok::exclaim},
1028                    {"=", tok::equal},
1029                    // FIXME:
1030                    // {"<", tok::less},
1031                    {">", tok::greater},
1032                    {"+=", tok::plusequal},
1033                    {"-=", tok::minusequal},
1034                    {"*=", tok::starequal},
1035                    {"/=", tok::slashequal},
1036                    {"%=", tok::percentequal},
1037                    {"^=", tok::caretequal},
1038                    {"&=", tok::ampequal},
1039                    {"|=", tok::pipeequal},
1040                    {"<<", tok::lessless},
1041                    {">>", tok::greatergreater},
1042                    {">>=", tok::greatergreaterequal},
1043                    {"<<=", tok::lesslessequal},
1044                    {"==", tok::equalequal},
1045                    {"!=", tok::exclaimequal},
1046                    {"<=", tok::lessequal},
1047                    {">=", tok::greaterequal},
1048                    {"<=>", tok::spaceship},
1049                    {"&&", tok::ampamp},
1050                    {"||", tok::pipepipe},
1051                    {"++", tok::plusplus},
1052                    {"--", tok::minusminus},
1053                    {",", tok::comma},
1054                    {"->*", tok::arrowstar},
1055                    {"->", tok::arrow}};
1056 
1057   for (const auto &Operator : Operators) {
1058     std::string Input("C<&operator");
1059     Input += Operator.Text;
1060     Input += " > a;";
1061     auto Tokens = annotate(std::string(Input));
1062     ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1063     EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
1064     EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator);
1065     EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
1066   }
1067 
1068   auto Tokens = annotate("C<&operator< <X>> lt;");
1069   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1070   EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
1071   EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator);
1072   EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
1073   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1074   EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
1075 }
1076 
1077 TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
1078   auto Tokens = annotate("template <typename T>\n"
1079                          "concept C = (Foo && Bar) && (Bar && Baz);");
1080 
1081   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1082   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
1083   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1084   EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
1085 
1086   Tokens = annotate("template <typename T>\n"
1087                     "concept C = Foo && !Bar;");
1088 
1089   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1090   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
1091   EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);
1092 
1093   Tokens = annotate("template <typename T>\n"
1094                     "concept C = requires(T t) {\n"
1095                     "  { t.foo() };\n"
1096                     "} && Bar<T> && Baz<T>;");
1097   ASSERT_EQ(Tokens.size(), 35u) << Tokens;
1098   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1099   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1100   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1101   EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
1102   EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
1103 
1104   Tokens = annotate("template<typename T>\n"
1105                     "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
1106                     "struct Foo;");
1107   ASSERT_EQ(Tokens.size(), 36u) << Tokens;
1108   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1109   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1110   EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
1111   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
1112   EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
1113   EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
1114   EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
1115   EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
1116   EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
1117   EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
1118 
1119   Tokens =
1120       annotate("template<typename T>\n"
1121                "requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
1122                "struct Foo;");
1123   ASSERT_EQ(Tokens.size(), 38u) << Tokens;
1124   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1125   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
1126   EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
1127   EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
1128   EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
1129   EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
1130   EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
1131   EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
1132   EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
1133   EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
1134   EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
1135 
1136   Tokens = annotate("template <typename T>\n"
1137                     "void foo(T) noexcept requires Bar<T>;");
1138   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1139   EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
1140 
1141   Tokens = annotate("template <typename T>\n"
1142                     "requires Bar<T> || Baz<T>\n"
1143                     "auto foo(T) -> int;");
1144   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1145   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1146   EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);
1147   EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
1148   EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);
1149 
1150   Tokens = annotate("template <typename T>\n"
1151                     "requires Bar<T>\n"
1152                     "bool foo(T) { return false; }");
1153   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1154   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1155   EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
1156   EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
1157   EXPECT_TOKEN(Tokens[12], tok::l_paren, TT_FunctionDeclarationLParen);
1158 
1159   Tokens = annotate("template <typename T>\n"
1160                     "requires Bar<T>\n"
1161                     "decltype(auto) foo(T) { return false; }");
1162   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1163   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1164   EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
1165   EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);
1166   EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_FunctionDeclarationLParen);
1167 
1168   Tokens = annotate("template <typename T>\n"
1169                     "struct S {\n"
1170                     "  void foo() const requires Bar<T>;\n"
1171                     "  void bar() const & requires Baz<T>;\n"
1172                     "  void bar() && requires Baz2<T>;\n"
1173                     "  void baz() const & noexcept requires Baz<T>;\n"
1174                     "  void baz() && noexcept requires Baz2<T>;\n"
1175                     "};\n"
1176                     "\n"
1177                     "void S::bar() const & requires Baz<T> { }");
1178   ASSERT_EQ(Tokens.size(), 85u) << Tokens;
1179   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1180   EXPECT_TOKEN(Tokens[24], tok::amp, TT_PointerOrReference);
1181   EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
1182   EXPECT_TOKEN(Tokens[35], tok::ampamp, TT_PointerOrReference);
1183   EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
1184   EXPECT_TOKEN(Tokens[47], tok::amp, TT_PointerOrReference);
1185   EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
1186   EXPECT_TOKEN(Tokens[59], tok::ampamp, TT_PointerOrReference);
1187   EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
1188   EXPECT_TOKEN(Tokens[76], tok::amp, TT_PointerOrReference);
1189   EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
1190 
1191   Tokens = annotate("void Class::member() && requires(Constant) {}");
1192   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1193   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1194 
1195   Tokens = annotate("void Class::member() && requires(Constant<T>) {}");
1196   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1197   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1198 
1199   Tokens =
1200       annotate("void Class::member() && requires(Namespace::Constant<T>) {}");
1201   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1202   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1203 
1204   Tokens = annotate("void Class::member() && requires(typename "
1205                     "Namespace::Outer<T>::Inner::Constant) {}");
1206   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1207   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1208 
1209   Tokens = annotate("struct [[nodiscard]] zero_t {\n"
1210                     "  template<class T>\n"
1211                     "    requires requires { number_zero_v<T>; }\n"
1212                     "  [[nodiscard]] constexpr operator T() const { "
1213                     "return number_zero_v<T>; }\n"
1214                     "};");
1215   ASSERT_EQ(Tokens.size(), 44u) << Tokens;
1216   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1217   EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
1218   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
1219   EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
1220   EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
1221   EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
1222 
1223   Tokens =
1224       annotate("template <class A, class B> concept C ="
1225                "std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
1226   ASSERT_EQ(Tokens.size(), 31u) << Tokens;
1227   EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
1228   EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
1229   EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
1230   EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);
1231   EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);
1232   EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);
1233   EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);
1234 
1235   Tokens = annotate("auto bar() -> int requires(is_integral_v<T>) {}");
1236   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1237   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1238 
1239   Tokens = annotate("auto bar() -> void requires(is_integral_v<T>) {}");
1240   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1241   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1242 
1243   Tokens = annotate("auto bar() -> MyType requires(is_integral_v<T>) {}");
1244   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1245   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1246 
1247   Tokens =
1248       annotate("auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");
1249   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1250   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1251 
1252   Tokens =
1253       annotate("auto bar() -> qualified::type requires(is_integral_v<T>) {}");
1254   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1255   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
1256 
1257   Tokens =
1258       annotate("auto bar() -> Template<type> requires(is_integral_v<T>) {}");
1259   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1260   EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);
1261 
1262   Tokens = annotate("void foo() requires((A<T>) && C) {}");
1263   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1264   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1265   EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
1266 
1267   Tokens = annotate("void foo() requires(((A<T>) && C)) {}");
1268   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1269   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1270   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1271 
1272   Tokens = annotate("void foo() requires([](T&&){}(t)) {}");
1273   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1274   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1275   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1276 
1277   Tokens = annotate("void foo() requires([](T&& u){}(t)) {}");
1278   ASSERT_EQ(Tokens.size(), 22u) << Tokens;
1279   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1280   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1281 
1282   Tokens = annotate("void f() & requires(true) {}");
1283   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1284   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1285   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1286 
1287   Tokens = annotate("void f() & requires(C<true, true>) {}");
1288   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1289   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1290   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1291 
1292   Tokens = annotate("template <typename T>\n"
1293                     "concept C = (!Foo<T>) && Bar;");
1294   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1295   EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
1296 
1297   Tokens = annotate("void f() & requires(C<decltype(x)>) {}");
1298   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1299   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1300   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1301 
1302   Tokens = annotate("auto f() -> int& requires(C<decltype(x)>) {}");
1303   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1304   EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1305   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1306 
1307   Tokens = annotate("bool x = t && requires(decltype(t) x) { x.foo(); };");
1308   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1309   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1310 
1311   Tokens = annotate("bool x = t && requires(Foo<decltype(t)> x) { x.foo(); };");
1312   ASSERT_EQ(Tokens.size(), 26u) << Tokens;
1313   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1314 
1315   Tokens = annotate("bool x = t && requires(Foo<C1 || C2> x) { x.foo(); };");
1316   ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1317   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1318 }
1319 
1320 TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
1321   auto Tokens = annotate("bool b = requires(int i) { i + 5; };");
1322   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1323   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1324   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1325   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1326 
1327   Tokens = annotate("if (requires(int i) { i + 5; }) return;");
1328   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1329   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1330   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1331   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
1332 
1333   Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;");
1334   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1335   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
1336   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
1337   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
1338 
1339   Tokens = annotate("foo(requires(const T t) {});");
1340   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1341   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1342   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1343   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1344 
1345   Tokens = annotate("foo(requires(const int t) {});");
1346   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1347   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1348   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1349   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1350 
1351   Tokens = annotate("foo(requires(const T t) {});");
1352   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1353   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1354   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1355   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1356 
1357   Tokens = annotate("foo(requires(int const* volatile t) {});");
1358   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1359   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1360   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1361   EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1362   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1363 
1364   Tokens = annotate("foo(requires(T const* volatile t) {});");
1365   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1366   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1367   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1368   EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1369   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1370 
1371   Tokens = annotate("foo(requires(T& t) {});");
1372   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1373   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1374   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1375   EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);
1376   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1377 
1378   Tokens = annotate("foo(requires(T&& t) {});");
1379   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1380   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1381   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1382   EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);
1383   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1384 
1385   Tokens = annotate("bool foo = requires(T& t) {};");
1386   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1387   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1388   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1389   EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1390   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1391 
1392   Tokens = annotate("bool foo = requires(T&& t) {};");
1393   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1394   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1395   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1396   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
1397   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1398 
1399   Tokens =
1400       annotate("foo(requires(const typename Outer<T>::Inner * const t) {});");
1401   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1402   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1403   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1404   EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
1405   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
1406 
1407   Tokens = annotate("template <typename T>\n"
1408                     "concept C = requires(T T) {\n"
1409                     "  requires Bar<T> && Foo<T>;\n"
1410                     "};");
1411   ASSERT_EQ(Tokens.size(), 28u) << Tokens;
1412   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1413   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1414   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1415   EXPECT_TOKEN(Tokens[14], tok::kw_requires,
1416                TT_RequiresClauseInARequiresExpression);
1417 
1418   Tokens = annotate("template <typename T>\n"
1419                     "concept C = requires(T T) {\n"
1420                     "  { t.func() } -> std::same_as<int>;"
1421                     "  requires Bar<T> && Foo<T>;\n"
1422                     "};");
1423   ASSERT_EQ(Tokens.size(), 43u) << Tokens;
1424   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1425   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1426   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1427   EXPECT_TOKEN(Tokens[29], tok::kw_requires,
1428                TT_RequiresClauseInARequiresExpression);
1429 
1430   // Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
1431   Tokens = annotate("bool r10 = requires (struct new_struct { int x; } s) { "
1432                     "requires true; };");
1433   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1434   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1435   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1436   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
1437 
1438   Tokens = annotate("bool foo = requires(C<true, true> c) {\n"
1439                     "  { c.foo(); }\n"
1440                     "};");
1441   ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1442   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1443   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1444   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1445 }
1446 
1447 TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {
1448   // Everything after #pragma region should be ImplicitStringLiteral
1449   auto Tokens = annotate("#pragma region Foo(Bar: Hello)");
1450   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1451   EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);
1452   EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);
1453   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);
1454 
1455   // Make sure it's annotated correctly inside a function as well
1456   Tokens = annotate("void test(){\n#pragma region Foo(Bar: Hello)\n}");
1457   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1458   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);
1459   EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);
1460   EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);
1461 }
1462 
1463 TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
1464   const char *BaseCode = nullptr;
1465   const char *ConstrainedCode = nullptr;
1466   auto BaseTokenCount = 0u;
1467   auto RequiresTokenCount = 0u;
1468   auto PrefixTokenCount = 0u;
1469 
1470   auto TestRequires = [&](int Line) {
1471     const auto BaseTokens = annotate(BaseCode);
1472     const auto ConstrainedTokens = annotate(ConstrainedCode);
1473 
1474 #define LINE " (Line " << Line << ')'
1475 
1476     ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;
1477     ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)
1478         << LINE;
1479 
1480     for (auto I = 0u; I < BaseTokenCount; ++I) {
1481       EXPECT_EQ(
1482           *BaseTokens[I],
1483           *ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])
1484           << I << LINE;
1485     }
1486 
1487 #undef LINE
1488   };
1489 
1490   BaseCode = "template<typename T>\n"
1491              "T Pi = 3.14;";
1492   ConstrainedCode = "template<typename T>\n"
1493                     "  requires Foo<T>\n"
1494                     "T Pi = 3.14;";
1495   BaseTokenCount = 11;
1496   RequiresTokenCount = 5;
1497   PrefixTokenCount = 5;
1498   TestRequires(__LINE__);
1499 
1500   BaseCode = "template<typename T>\n"
1501              "struct Bar;";
1502   ConstrainedCode = "template<typename T>\n"
1503                     "  requires Foo<T>\n"
1504                     "struct Bar;";
1505   BaseTokenCount = 9;
1506   TestRequires(__LINE__);
1507 
1508   BaseCode = "template<typename T>\n"
1509              "struct Bar {\n"
1510              "  T foo();\n"
1511              "  T bar();\n"
1512              "};";
1513   ConstrainedCode = "template<typename T>\n"
1514                     "  requires Foo<T>\n"
1515                     "struct Bar {\n"
1516                     "  T foo();\n"
1517                     "  T bar();\n"
1518                     "};";
1519   BaseTokenCount = 21;
1520   TestRequires(__LINE__);
1521 
1522   BaseCode = "template<typename T>\n"
1523              "Bar(T) -> Bar<T>;";
1524   ConstrainedCode = "template<typename T>\n"
1525                     "  requires Foo<T>\n"
1526                     "Bar(T) -> Bar<T>;";
1527   BaseTokenCount = 16;
1528   TestRequires(__LINE__);
1529 
1530   BaseCode = "template<typename T>\n"
1531              "T foo();";
1532   ConstrainedCode = "template<typename T>\n"
1533                     "  requires Foo<T>\n"
1534                     "T foo();";
1535   BaseTokenCount = 11;
1536   TestRequires(__LINE__);
1537 
1538   BaseCode = "template<typename T>\n"
1539              "T foo() {\n"
1540              "  auto bar = baz();\n"
1541              "  return bar + T{};\n"
1542              "}";
1543   ConstrainedCode = "template<typename T>\n"
1544                     "  requires Foo<T>\n"
1545                     "T foo() {\n"
1546                     "  auto bar = baz();\n"
1547                     "  return bar + T{};\n"
1548                     "}";
1549   BaseTokenCount = 26;
1550   TestRequires(__LINE__);
1551 
1552   BaseCode = "template<typename T>\n"
1553              "T foo();";
1554   ConstrainedCode = "template<typename T>\n"
1555                     "T foo() requires Foo<T>;";
1556   BaseTokenCount = 11;
1557   PrefixTokenCount = 9;
1558   TestRequires(__LINE__);
1559 
1560   BaseCode = "template<typename T>\n"
1561              "T foo() {\n"
1562              "  auto bar = baz();\n"
1563              "  return bar + T{};\n"
1564              "}";
1565   ConstrainedCode = "template<typename T>\n"
1566                     "T foo() requires Foo<T> {\n"
1567                     "  auto bar = baz();\n"
1568                     "  return bar + T{};\n"
1569                     "}";
1570   BaseTokenCount = 26;
1571   TestRequires(__LINE__);
1572 
1573   BaseCode = "template<typename T>\n"
1574              "T foo();";
1575   ConstrainedCode = "template<typename T>\n"
1576                     "  requires(Foo<T>)\n"
1577                     "T foo();";
1578   BaseTokenCount = 11;
1579   RequiresTokenCount = 7;
1580   PrefixTokenCount = 5;
1581   TestRequires(__LINE__);
1582 
1583   BaseCode = "template<typename T>\n"
1584              "Bar(T) -> Bar<typename T::I>;";
1585   ConstrainedCode = "template<typename T>\n"
1586                     "  requires requires(T &&t) {\n"
1587                     "             typename T::I;\n"
1588                     "           }\n"
1589                     "Bar(T) -> Bar<typename T::I>;";
1590   BaseTokenCount = 19;
1591   RequiresTokenCount = 14;
1592   PrefixTokenCount = 5;
1593   TestRequires(__LINE__);
1594 
1595   BaseCode = "struct [[nodiscard]] zero_t {\n"
1596              "  template<class T>\n"
1597              "  [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1598              "};";
1599   ConstrainedCode =
1600       "struct [[nodiscard]] zero_t {\n"
1601       "  template<class T>\n"
1602       "    requires requires { v<T>; }\n"
1603       "  [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1604       "};";
1605   BaseTokenCount = 35;
1606   RequiresTokenCount = 9;
1607   PrefixTokenCount = 13;
1608   TestRequires(__LINE__);
1609 
1610   BaseCode = "constexpr Foo(Foo const &other)\n"
1611              "    : value{other.value} {\n"
1612              "  do_magic();\n"
1613              "  do_more_magic();\n"
1614              "}";
1615   ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1616                     "  requires std::is_copy_constructible<T>\n"
1617                     "    : value{other.value} {\n"
1618                     "  do_magic();\n"
1619                     "  do_more_magic();\n"
1620                     "}";
1621   BaseTokenCount = 26;
1622   RequiresTokenCount = 7;
1623   PrefixTokenCount = 8;
1624   TestRequires(__LINE__);
1625 
1626   BaseCode = "constexpr Foo(Foo const &other)\n"
1627              "    : value{other.value} {\n"
1628              "  do_magic();\n"
1629              "  do_more_magic();\n"
1630              "}";
1631   ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1632                     "  requires (std::is_copy_constructible<T>)\n"
1633                     "    : value{other.value} {\n"
1634                     "  do_magic();\n"
1635                     "  do_more_magic();\n"
1636                     "}";
1637   RequiresTokenCount = 9;
1638   TestRequires(__LINE__);
1639 
1640   BaseCode = "template<typename T>\n"
1641              "ANNOTATE(\"S\"\n"
1642              "         \"S\")\n"
1643              "void foo();";
1644   ConstrainedCode = "template<typename T>\n"
1645                     "  requires(true)\n"
1646                     "ANNOTATE(\"S\"\n"
1647                     "         \"S\")\n"
1648                     "void foo();";
1649   BaseTokenCount = 16;
1650   RequiresTokenCount = 4;
1651   PrefixTokenCount = 5;
1652   TestRequires(__LINE__);
1653 }
1654 
1655 TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
1656   auto Tokens = annotate("__asm{\n"
1657                          "\"a\":\n"
1658                          ": x\n"
1659                          ":};");
1660   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1661   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1662   EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
1663   EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1664   EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1665   EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1666   EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_InlineASMBrace);
1667 
1668   Tokens = annotate("__asm(\n"
1669                     "\"a\":\n"
1670                     ": x\n"
1671                     ":);");
1672   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1673   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1674   EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1675   EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1676   EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1677 
1678   Tokens = annotate("asm volatile (\n"
1679                     "\"a_label:\"\n"
1680                     ":\n"
1681                     ": x\n"
1682                     ":);");
1683   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1684   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1685   EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1686   EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1687   EXPECT_TOKEN(Tokens[7], tok::colon, TT_InlineASMColon);
1688 
1689   Tokens = annotate("__asm__(\n"
1690                     "\"a_label:\"\n"
1691                     ": x\n"
1692                     ":\n"
1693                     ": y);");
1694   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1695   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1696   EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1697   EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1698   EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1699 
1700   Tokens = annotate("__asm volatile (\n"
1701                     "\"a_label:\"\n"
1702                     "\"a b c(%%x)\"\n"
1703                     ":\n"
1704                     ": x\n"
1705                     ":);");
1706   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1707   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1708   EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1709   EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1710   EXPECT_TOKEN(Tokens[8], tok::colon, TT_InlineASMColon);
1711 
1712   Tokens = annotate("asm(\n"
1713                     "\"insn\"\n"
1714                     ": \"=r\" (var1), \"=&r\" (value)\n"
1715                     ":\n"
1716                     ": \"memory\");");
1717   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1718   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1719   EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1720   EXPECT_TOKEN(Tokens[13], tok::colon, TT_InlineASMColon);
1721   EXPECT_TOKEN(Tokens[14], tok::colon, TT_InlineASMColon);
1722 
1723   Tokens = annotate("__asm__ volatile (\n"
1724                     "\"ldr r1, [r0, %%[sym]]\"\n"
1725                     ":\n"
1726                     ": [sym] \"J\" (aaaaa(aaaa, aaaa))\n"
1727                     ");");
1728   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1729   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1730   EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1731   EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1732   EXPECT_TOKEN(Tokens[6], tok::l_square, TT_InlineASMSymbolicNameLSquare);
1733 }
1734 
1735 TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
1736   auto Tokens = annotate("int (^)() = ^ ()\n"
1737                          "  external_source_symbol() { //\n"
1738                          "  return 1;\n"
1739                          "};");
1740   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1741   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1742   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
1743 
1744   Tokens = annotate("int *p = ^int*(){ //\n"
1745                     "  return nullptr;\n"
1746                     "}();");
1747   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1748   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
1749 
1750   Tokens = annotate("id (^block)(Foo *a) = ^id _Nullable(Foo *_Nullable a) {\n"
1751                     "  return a;\n"
1752                     "};");
1753   ASSERT_EQ(Tokens.size(), 27u) << Tokens;
1754   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); // Not CtorDtorDeclName.
1755   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1756 }
1757 
1758 TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodExpr) {
1759   auto Tokens = annotate("void f() {\n"
1760                          "  //\n"
1761                          "  BOOL a = [b.c n] > 1;\n"
1762                          "}");
1763   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1764   EXPECT_TOKEN(Tokens[9], tok::l_square, TT_ObjCMethodExpr);
1765   EXPECT_TOKEN(Tokens[15], tok::greater, TT_BinaryOperator);
1766 }
1767 
1768 TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
1769   auto Tokens = annotate("[]() constexpr {}");
1770   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1771   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1772   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1773   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1774 
1775   Tokens = annotate("[]() consteval {}");
1776   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1777   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1778   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1779   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1780 
1781   Tokens = annotate("[]() mutable {}");
1782   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1783   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1784   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1785   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1786 
1787   Tokens = annotate("[]() static {}");
1788   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1789   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1790   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1791   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1792 
1793   Tokens = annotate("[]() -> auto {}");
1794   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1795   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1796   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1797   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1798   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1799 
1800   Tokens = annotate("[]() -> auto & {}");
1801   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1802   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1803   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1804   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1805   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1806 
1807   Tokens = annotate("[]() -> auto * {}");
1808   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1809   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1810   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1811   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1812   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1813 
1814   Tokens = annotate("[] {}");
1815   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1816   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1817   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
1818 
1819   Tokens = annotate("[] noexcept {}");
1820   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1821   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1822   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
1823 
1824   Tokens = annotate("[] -> auto {}");
1825   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1826   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1827   EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
1828   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
1829 
1830   Tokens = annotate("[] -> struct S { return {}; }");
1831   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1832   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1833   EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
1834   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1835 
1836   Tokens = annotate("foo([&](u32 bar) __attribute__((attr)) -> void {});");
1837   ASSERT_EQ(Tokens.size(), 22u) << Tokens;
1838   EXPECT_TOKEN(Tokens[2], tok::l_square, TT_LambdaLSquare);
1839   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_LambdaDefinitionLParen);
1840   EXPECT_TOKEN(Tokens[15], tok::arrow, TT_LambdaArrow);
1841   EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
1842 
1843   Tokens = annotate("[] <typename T> () {}");
1844   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1845   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1846   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1847   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
1848   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
1849 
1850   Tokens = annotate("[] <typename T> {}");
1851   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1852   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1853   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1854   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1855 
1856   Tokens = annotate("[] <typename... T> () {}");
1857   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1858   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1859   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1860   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
1861   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1862 
1863   Tokens = annotate("[] <typename... T> {}");
1864   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1865   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1866   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1867   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1868 
1869   Tokens = annotate("[] <int... T> () {}");
1870   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1871   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1872   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1873   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
1874   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1875 
1876   Tokens = annotate("[] <int... T> {}");
1877   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1878   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1879   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1880   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1881 
1882   Tokens = annotate("[] <Foo... T> () {}");
1883   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1884   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1885   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1886   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
1887   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1888 
1889   Tokens = annotate("[] <Foo... T> {}");
1890   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1891   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1892   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1893   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1894 
1895   // Lambdas with a requires-clause
1896   Tokens = annotate("[] <typename T> (T t) requires Bar<T> {}");
1897   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1898   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1899   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1900   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
1901   EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
1902   EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
1903   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1904 
1905   Tokens = annotate("[] <typename T> (T &&t) requires Bar<T> {}");
1906   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1907   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1908   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1909   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
1910   EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);
1911   EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
1912   EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
1913   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
1914 
1915   Tokens = annotate("[] <typename T> (T t) requires Foo<T> || Bar<T> {}");
1916   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1917   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1918   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1919   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
1920   EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
1921   EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
1922   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1923 
1924   Tokens = annotate("[] <typename T> (T t) -> T requires Bar<T> {}");
1925   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1926   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1927   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1928   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
1929   EXPECT_TOKEN(Tokens[10], tok::arrow, TT_LambdaArrow);
1930   EXPECT_TOKEN(Tokens[12], tok::kw_requires, TT_RequiresClause);
1931   EXPECT_TRUE(Tokens[16]->ClosesRequiresClause);
1932   EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
1933 
1934   Tokens = annotate("[] <typename T> requires Bar<T> (T t) {}");
1935   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1936   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1937   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1938   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1939   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1940   // FIXME:
1941   // EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
1942   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1943 
1944   Tokens = annotate("[] <typename T> requires Bar<T> (T &&t) {}");
1945   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1946   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1947   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1948   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1949   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1950   // FIXME:
1951   // EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
1952   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_PointerOrReference);
1953   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
1954 
1955   Tokens = annotate("[] <typename T> requires Foo<T> || Bar<T> (T t) {}");
1956   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1957   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1958   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1959   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1960   EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
1961   // FIXME:
1962   // EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_LambdaDefinitionLParen);
1963   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1964 
1965   Tokens = annotate("[] <typename T> requires true (T&& t) {}");
1966   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1967   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1968   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1969   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1970   EXPECT_TRUE(Tokens[7]->ClosesRequiresClause);
1971   // FIXME:
1972   // EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
1973   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1974   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
1975 
1976   Tokens = annotate("[] <typename T> requires Bar<T> {}");
1977   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1978   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1979   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1980   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1981   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1982   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_LambdaLBrace);
1983 
1984   Tokens = annotate("[] <typename T> requires Bar<T> noexcept {}");
1985   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1986   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1987   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1988   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1989   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1990   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
1991 
1992   Tokens = annotate("[] <typename T> requires Bar<T> -> T {}");
1993   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1994   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1995   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1996   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1997   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1998   EXPECT_TOKEN(Tokens[11], tok::arrow, TT_LambdaArrow);
1999   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2000 
2001   Tokens = annotate("[] <typename T> requires Foo<T> (T t) requires Bar<T> {}");
2002   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2003   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2004   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2005   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2006   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2007   // FIXME:
2008   // EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2009   EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);
2010   EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
2011   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2012 
2013   Tokens = annotate("[] <typename T = int> (T t) {}");
2014   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2015   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2016   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2017   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2018   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2019   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2020 
2021   Tokens = annotate("[] <int I = 0> (T t) {}");
2022   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2023   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2024   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2025   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2026   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2027   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2028 
2029   Tokens = annotate("[] <bool b = false> (T t) {}");
2030   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2031   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2032   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2033   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2034   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2035   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2036 
2037   Tokens = annotate("[] <bool b = true && false> (T&& t) {}");
2038   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2039   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2040   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2041   EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
2042   EXPECT_TOKEN(Tokens[9], tok::greater, TT_TemplateCloser);
2043   EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_LambdaDefinitionLParen);
2044   EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
2045   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2046 
2047   Tokens = annotate("[] <typename T = int> requires Foo<T> (T t) {}");
2048   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2049   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2050   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2051   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2052   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
2053   // FIXME:
2054   // EXPECT_TOKEN(Tokens[13], tok::l_paren, TT_LambdaDefinitionLParen);
2055   EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
2056 }
2057 
2058 TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
2059   auto Tokens = annotate("template <typename T>\n"
2060                          "DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
2061                          "string OldFunction(const string &parameter) {}");
2062   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2063   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);
2064 
2065   Tokens = annotate("template <typename T>\n"
2066                     "A(T) noexcept;");
2067   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2068   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2069 
2070   Tokens = annotate("FOO(bar)();");
2071   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2072   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
2073 }
2074 
2075 TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
2076   auto Tokens = annotate("void f [[noreturn]] ();");
2077   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2078   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2079   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2080 
2081   Tokens = annotate("void f [[noreturn]] () {}");
2082   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2083   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2084   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2085 
2086   Tokens = annotate("#define FOO Foo::\n"
2087                     "FOO Foo();");
2088   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2089   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_FunctionDeclarationName);
2090   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2091 
2092   Tokens = annotate("struct Foo {\n"
2093                     "  Bar (*func)();\n"
2094                     "};");
2095   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2096   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
2097   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
2098 
2099   Tokens = annotate("void instanceof();");
2100   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2101   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2102   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2103 
2104   Tokens = annotate("int iso_time(time_t);");
2105   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2106   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2107   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2108 
2109   auto Style = getLLVMStyle();
2110   Style.TypeNames.push_back("MyType");
2111   Tokens = annotate("int iso_time(MyType);", Style);
2112   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2113   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2114   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2115   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_TypeName);
2116 }
2117 
2118 TEST_F(TokenAnnotatorTest, UnderstandsCtorAndDtorDeclNames) {
2119   auto Tokens = annotate("class Foo { public: Foo(); };");
2120   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2121   EXPECT_TOKEN(Tokens[5], tok::identifier, TT_CtorDtorDeclName);
2122   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
2123 
2124   Tokens = annotate("class Foo { public: ~Foo(); };");
2125   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2126   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_CtorDtorDeclName);
2127   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2128 
2129   Tokens = annotate("struct Foo { [[deprecated]] Foo() {} };");
2130   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2131   EXPECT_TOKEN(Tokens[8], tok::identifier, TT_CtorDtorDeclName);
2132   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_FunctionDeclarationLParen);
2133   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
2134 
2135   Tokens = annotate("struct Foo { [[deprecated]] ~Foo() {} };");
2136   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2137   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
2138   EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);
2139   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2140 
2141   Tokens = annotate("struct Foo { Foo() [[deprecated]] {} };");
2142   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2143   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2144   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2145   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
2146 
2147   Tokens = annotate("struct Foo { ~Foo() [[deprecated]] {} };");
2148   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2149   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_CtorDtorDeclName);
2150   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
2151   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2152 
2153   Tokens = annotate("struct Foo { [[deprecated]] explicit Foo() {} };");
2154   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2155   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
2156   EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);
2157   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2158 
2159   Tokens = annotate("struct Foo { virtual [[deprecated]] ~Foo() {} };");
2160   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2161   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
2162   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
2163   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2164 
2165   Tokens = annotate("Foo::Foo() {}");
2166   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2167   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
2168   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
2169   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
2170 
2171   Tokens = annotate("Foo::~Foo() {}");
2172   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2173   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2174   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2175   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
2176 
2177   Tokens = annotate("struct Test {\n"
2178                     "  Test()\n"
2179                     "      : l([] {\n"
2180                     "          Short::foo();\n"
2181                     "        }) {}\n"
2182                     "};");
2183   ASSERT_EQ(Tokens.size(), 25u) << Tokens;
2184   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2185   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2186   EXPECT_TOKEN(Tokens[14], tok::identifier, TT_Unknown);
2187 }
2188 
2189 TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {
2190   auto Tokens = annotate("_Generic(x, int: 1, default: 0)");
2191   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2192   EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);
2193   EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);
2194   EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);
2195 }
2196 
2197 TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
2198   auto Tokens = annotate("auto f() -> int;");
2199   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2200   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2201 
2202   Tokens = annotate("auto operator->() -> int;");
2203   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2204   EXPECT_TOKEN(Tokens[2], tok::arrow, TT_OverloadedOperator);
2205   EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
2206 
2207   Tokens = annotate("auto operator++(int) -> int;");
2208   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2209   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
2210 
2211   Tokens = annotate("auto operator=() -> int;");
2212   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2213   EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
2214 
2215   Tokens = annotate("auto operator=(int) -> int;");
2216   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2217   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
2218 
2219   Tokens = annotate("auto foo() -> auto { return Val; }");
2220   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2221   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2222 
2223   Tokens = annotate("struct S { auto bar() const -> int; };");
2224   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2225   EXPECT_TOKEN(Tokens[8], tok::arrow, TT_TrailingReturnArrow);
2226 
2227   // Not trailing return arrows
2228   Tokens = annotate("auto a = b->c;");
2229   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2230   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2231 
2232   Tokens = annotate("auto a = (b)->c;");
2233   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2234   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
2235 
2236   Tokens = annotate("auto a = b()->c;");
2237   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2238   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
2239 
2240   Tokens = annotate("auto a = b->c();");
2241   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2242   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2243 
2244   Tokens = annotate("decltype(auto) a = b()->c;");
2245   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2246   EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
2247 
2248   Tokens = annotate("void f() { auto a = b->c(); }");
2249   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2250   EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
2251 
2252   Tokens = annotate("void f() { auto a = b()->c; }");
2253   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2254   EXPECT_TOKEN(Tokens[11], tok::arrow, TT_Unknown);
2255 
2256   Tokens = annotate("#define P(ptr) auto p = (ptr)->p");
2257   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2258   EXPECT_TOKEN(Tokens[12], tok::arrow, TT_Unknown);
2259 
2260   Tokens = annotate("void f() FOO(foo->bar);");
2261   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2262   EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
2263 
2264   Tokens = annotate("__attribute__((cold)) C() : Base(obj->func()) {}");
2265   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
2266   EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
2267 
2268   auto Style = getLLVMStyle();
2269   Style.StatementAttributeLikeMacros.push_back("emit");
2270   Tokens = annotate("emit foo()->bar;", Style);
2271   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2272   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
2273   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2274 
2275   // Mixed
2276   Tokens = annotate("auto f() -> int { auto a = b()->c; }");
2277   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2278   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2279   EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
2280 }
2281 
2282 TEST_F(TokenAnnotatorTest, UnderstandHashInMacro) {
2283   auto Tokens = annotate("#define Foo(Bar) \\\n"
2284                          "  { \\\n"
2285                          "    #Bar \\\n"
2286                          "  }");
2287   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2288   EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2289   EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2290 
2291   Tokens = annotate("#define Foo(Bar) \\\n"
2292                     "  { #Bar }");
2293   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2294   EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2295   EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2296 
2297   Tokens = annotate("#define FOO(typeName, realClass) \\\n"
2298                     "  {#typeName, foo<Foo>(new foo<realClass>(#typeName))}");
2299   ASSERT_EQ(Tokens.size(), 29u) << Tokens;
2300   EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
2301   EXPECT_BRACE_KIND(Tokens[27], BK_BracedInit);
2302 }
2303 
2304 TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacros) {
2305   // '__attribute__' has special handling.
2306   auto Tokens = annotate("__attribute__((X)) void Foo(void);");
2307   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2308   EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2309   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2310   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2311 
2312   // Generic macro has no special handling in this location.
2313   Tokens = annotate("A(X) void Foo(void);");
2314   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2315   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2316   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_Unknown);
2317 
2318   // Add a custom AttributeMacro. Test that it has the same behavior.
2319   FormatStyle Style = getLLVMStyle();
2320   Style.AttributeMacros.push_back("A");
2321 
2322   // An "AttributeMacro" gets annotated like '__attribute__'.
2323   Tokens = annotate("A(X) void Foo(void);", Style);
2324   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2325   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2326   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2327   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2328 }
2329 
2330 TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCDecl) {
2331   // '__attribute__' has special handling.
2332   auto Tokens = annotate("__attribute__((X)) @interface Foo");
2333   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2334   EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2335   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2336   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2337 
2338   // Generic macro has no special handling in this location.
2339   Tokens = annotate("A(X) @interface Foo");
2340   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2341   // Note: Don't check token-type as a random token in this position is hard to
2342   // reason about.
2343   EXPECT_TOKEN_KIND(Tokens[0], tok::identifier);
2344   EXPECT_TOKEN_KIND(Tokens[1], tok::l_paren);
2345 
2346   // Add a custom AttributeMacro. Test that it has the same behavior.
2347   FormatStyle Style = getLLVMStyle();
2348   Style.AttributeMacros.push_back("A");
2349 
2350   // An "AttributeMacro" gets annotated like '__attribute__'.
2351   Tokens = annotate("A(X) @interface Foo", Style);
2352   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2353   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2354   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2355   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2356 }
2357 
2358 TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCMethodDecl) {
2359   // '__attribute__' has special handling.
2360   auto Tokens = annotate("- (id)init __attribute__((X));");
2361   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2362   EXPECT_TOKEN(Tokens[5], tok::kw___attribute, TT_Unknown);
2363   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2364   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2365 
2366   // Generic macro has no special handling in this location.
2367   Tokens = annotate("- (id)init A(X);");
2368   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2369   // Note: Don't check token-type as a random token in this position is hard to
2370   // reason about.
2371   EXPECT_TOKEN_KIND(Tokens[5], tok::identifier);
2372   EXPECT_TOKEN_KIND(Tokens[6], tok::l_paren);
2373 
2374   // Add a custom AttributeMacro. Test that it has the same behavior.
2375   FormatStyle Style = getLLVMStyle();
2376   Style.AttributeMacros.push_back("A");
2377 
2378   // An "AttributeMacro" gets annotated like '__attribute__'.
2379   Tokens = annotate("- (id)init A(X);", Style);
2380   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2381   EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
2382   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2383   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
2384 }
2385 
2386 TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCProperty) {
2387   // '__attribute__' has special handling.
2388   auto Tokens = annotate("@property(weak) id delegate __attribute__((X));");
2389   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2390   EXPECT_TOKEN(Tokens[7], tok::kw___attribute, TT_Unknown);
2391   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2392   EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_AttributeRParen);
2393 
2394   // Generic macro has no special handling in this location.
2395   Tokens = annotate("@property(weak) id delegate A(X);");
2396   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2397   // Note: Don't check token-type as a random token in this position is hard to
2398   // reason about.
2399   EXPECT_TOKEN_KIND(Tokens[7], tok::identifier);
2400   EXPECT_TOKEN_KIND(Tokens[8], tok::l_paren);
2401 
2402   // Add a custom AttributeMacro. Test that it has the same behavior.
2403   FormatStyle Style = getLLVMStyle();
2404   Style.AttributeMacros.push_back("A");
2405 
2406   // An "AttributeMacro" gets annotated like '__attribute__'.
2407   Tokens = annotate("@property(weak) id delegate A(X);", Style);
2408   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2409   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_AttributeMacro);
2410   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2411   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2412 }
2413 
2414 TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
2415   auto Annotate = [this](StringRef Code) {
2416     return annotate(Code, getLLVMStyle(FormatStyle::LK_Verilog));
2417   };
2418   // Test that unary operators get labeled as such and that operators like '++'
2419   // don't get split.
2420   tok::TokenKind Unary[] = {tok::plus,  tok::minus,    tok::exclaim,
2421                             tok::tilde, tok::amp,      tok::pipe,
2422                             tok::caret, tok::plusplus, tok::minusminus};
2423   for (auto Kind : Unary) {
2424     auto Tokens =
2425         Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");
2426     ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2427     EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);
2428   }
2429   // Operators formed by joining two operators like '^~'. For some of these
2430   // joined operators, we don't have a separate type, so we only test for their
2431   // precedence.
2432   std::pair<prec::Level, std::string> JoinedBinary[] = {
2433       {prec::Comma, "->"},        {prec::Comma, "<->"},
2434       {prec::Assignment, "+="},   {prec::Assignment, "-="},
2435       {prec::Assignment, "*="},   {prec::Assignment, "/="},
2436       {prec::Assignment, "%="},   {prec::Assignment, "&="},
2437       {prec::Assignment, "^="},   {prec::Assignment, "<<="},
2438       {prec::Assignment, ">>="},  {prec::Assignment, "<<<="},
2439       {prec::Assignment, ">>>="}, {prec::LogicalOr, "||"},
2440       {prec::LogicalAnd, "&&"},   {prec::Equality, "=="},
2441       {prec::Equality, "!="},     {prec::Equality, "==="},
2442       {prec::Equality, "!=="},    {prec::Equality, "==?"},
2443       {prec::Equality, "!=?"},    {prec::ExclusiveOr, "~^"},
2444       {prec::ExclusiveOr, "^~"},
2445   };
2446   for (auto Operator : JoinedBinary) {
2447     auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");
2448     ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2449     EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);
2450     EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);
2451   }
2452   // '~^' and '^~' can be unary as well as binary operators.
2453   auto Tokens = Annotate("x = ~^x;");
2454   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2455   EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2456   Tokens = Annotate("x = ^~x;");
2457   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2458   EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2459   // The unary operators '~&' and '~|' can only be unary operators. The current
2460   // implementation treats each of them as separate unary '~' and '&' or '|'
2461   // operators, which is enough for formatting purposes. In FormatTestVerilog,
2462   // there is a test that there is no space in between. And even if a new line
2463   // is inserted between the '~' and '|', the semantic meaning is the same as
2464   // the joined operator, so the CanBreakBefore property doesn't need to be
2465   // false for the second operator.
2466   Tokens = Annotate("x = ~&x;");
2467   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2468   EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2469   EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);
2470   Tokens = Annotate("x = ~|x;");
2471   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2472   EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2473   EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);
2474   // Test for block label colons.
2475   Tokens = Annotate("begin : x\n"
2476                     "end : x");
2477   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2478   EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);
2479   EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);
2480   // Test that the dimension colon is annotated correctly.
2481   Tokens = Annotate("var [1 : 0] x;");
2482   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2483   EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);
2484   Tokens = Annotate("extern function [1 : 0] x;");
2485   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2486   EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
2487   Tokens = Annotate("module test\n"
2488                     "    (input wire [7 : 0] a[7 : 0]);\n"
2489                     "endmodule");
2490   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2491   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);
2492   EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);
2493   EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);
2494   // Test case labels and ternary operators.
2495   Tokens = Annotate("case (x)\n"
2496                     "  x:\n"
2497                     "    x;\n"
2498                     "endcase");
2499   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2500   EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);
2501   Tokens = Annotate("case (x)\n"
2502                     "  x ? x : x:\n"
2503                     "    x;\n"
2504                     "endcase");
2505   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2506   EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
2507   EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
2508   EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);
2509   // Non-blocking assignments.
2510   Tokens = Annotate("a <= b;");
2511   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2512   EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2513   EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2514   Tokens = Annotate("if (a <= b) break;");
2515   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2516   EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2517   EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2518   Tokens = Annotate("a <= b <= a;");
2519   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2520   EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2521   EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2522   EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2523   EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2524 
2525   // Port lists in module instantiation.
2526   Tokens = Annotate("module_x instance_1(port_1), instance_2(port_2);");
2527   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2528   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2529   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_VerilogInstancePortLParen);
2530   Tokens = Annotate("module_x #(parameter) instance_1(port_1), "
2531                     "instance_2(port_2);");
2532   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2533   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2534   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogInstancePortLParen);
2535   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_VerilogInstancePortLParen);
2536 
2537   // Condition parentheses.
2538   Tokens = Annotate("assert (x);");
2539   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2540   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2541   Tokens = Annotate("assert #0 (x);");
2542   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2543   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_ConditionLParen);
2544   Tokens = Annotate("assert final (x);");
2545   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2546   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2547   Tokens = Annotate("foreach (x[x]) continue;");
2548   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2549   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2550   Tokens = Annotate("repeat (x[x]) continue;");
2551   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2552   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2553   Tokens = Annotate("case (x) endcase;");
2554   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2555   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2556 
2557   // Sensitivity list. The TT_Unknown type is clearly not binding for the
2558   // future, please adapt if those tokens get annotated.  This test is only here
2559   // to prevent the comma from being annotated as TT_VerilogInstancePortComma.
2560   Tokens = Annotate("always @(posedge x, posedge y);");
2561   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2562   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_Unknown);
2563   EXPECT_TOKEN(Tokens[5], tok::comma, TT_Unknown);
2564   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2565 
2566   // String literals in concatenation.
2567   Tokens = Annotate("x = {\"\"};");
2568   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2569   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2570   Tokens = Annotate("x = {\"\", \"\"};");
2571   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2572   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2573   EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2574   Tokens = Annotate("x = '{{\"\"}};");
2575   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2576   EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2577   // Cases where the string should not be annotated that type.  Fix the
2578   // `TT_Unknown` if needed in the future.
2579   Tokens = Annotate("x = {\"\" == \"\"};");
2580   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2581   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_Unknown);
2582   EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_Unknown);
2583   Tokens = Annotate("x = {(\"\")};");
2584   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2585   EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2586   Tokens = Annotate("x = '{\"\"};");
2587   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2588   EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2589 }
2590 
2591 TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) {
2592   auto Style = getLLVMStyle(FormatStyle::LK_TableGen);
2593   ASSERT_TRUE(Style.isTableGen());
2594 
2595   TestLexer Lexer(Allocator, Buffers, Style);
2596   AdditionalKeywords Keywords(Lexer.IdentTable);
2597   auto Annotate = [&Lexer](StringRef Code) { return Lexer.annotate(Code); };
2598 
2599   // Additional keywords representation test.
2600   auto Tokens = Annotate("def foo : Bar<1>;");
2601   ASSERT_TRUE(Keywords.isTableGenKeyword(*Tokens[0]));
2602   ASSERT_TRUE(Keywords.isTableGenDefinition(*Tokens[0]));
2603   ASSERT_TRUE(Tokens[0]->is(Keywords.kw_def));
2604   ASSERT_TRUE(Tokens[1]->is(TT_StartOfName));
2605 
2606   // Code, the multiline string token.
2607   Tokens = Annotate("[{ code is multiline string }]");
2608   ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2609   EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2610   EXPECT_FALSE(Tokens[0]->IsMultiline);
2611   // Case with multiple lines.
2612   Tokens = Annotate("[{ It can break\n"
2613                     "   across lines and the line breaks\n"
2614                     "   are retained in \n"
2615                     "   the string. }]");
2616   ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2617   EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2618   EXPECT_EQ(Tokens[0]->ColumnWidth, sizeof("[{ It can break\n") - 1);
2619   EXPECT_TRUE(Tokens[0]->IsMultiline);
2620   EXPECT_EQ(Tokens[0]->LastLineColumnWidth, sizeof("   the string. }]") - 1);
2621 
2622   // Numeric literals.
2623   Tokens = Annotate("1234");
2624   EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2625   Tokens = Annotate("-1");
2626   EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2627   Tokens = Annotate("+1234");
2628   EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2629   Tokens = Annotate("0b0110");
2630   EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2631   Tokens = Annotate("0x1abC");
2632   EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2633 
2634   // Identifier tokens. In TableGen, identifiers can begin with a number.
2635   // In ambiguous cases, the lexer tries to lex it as a number.
2636   // Even if the try fails, it does not fall back to identifier lexing and
2637   // regard as an error.
2638   // The ambiguity is not documented. The result of those tests are based on the
2639   // implementation of llvm::TGLexer::LexToken.
2640   // This is invalid syntax of number, but not an identifier.
2641   Tokens = Annotate("0x1234x");
2642   EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2643   Tokens = Annotate("identifier");
2644   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2645   // Identifier beginning with a number.
2646   Tokens = Annotate("0x");
2647   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2648   Tokens = Annotate("2dVector");
2649   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2650   Tokens = Annotate("01234Vector");
2651   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2652 
2653   // Structured statements.
2654   Tokens = Annotate("class Foo {}");
2655   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_FunctionLBrace);
2656   Tokens = Annotate("def Def: Foo {}");
2657   EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2658   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2659   Tokens = Annotate("if cond then {} else {}");
2660   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ControlStatementLBrace);
2661   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_ElseLBrace);
2662   Tokens = Annotate("defset Foo Def2 = {}");
2663   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2664 
2665   // Bang Operators.
2666   Tokens = Annotate("!foreach");
2667   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2668   Tokens = Annotate("!if");
2669   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2670   Tokens = Annotate("!cond");
2671   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2672 
2673   auto AnnotateValue = [this, &Style](StringRef Code) {
2674     // Values are annotated only in specific context.
2675     auto Result = annotate(("def X { let V = " + Code + "; }").str(), Style);
2676     return decltype(Result){Result.begin() + 6, Result.end() - 3};
2677   };
2678   // Both of bang/cond operators.
2679   Tokens = AnnotateValue("!cond(!eq(x, 0): 1, true: x)");
2680   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2681   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2682   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TableGenBangOperator);
2683   EXPECT_TOKEN(Tokens[8], tok::colon, TT_TableGenCondOperatorColon);
2684   EXPECT_TOKEN(Tokens[10], tok::comma, TT_TableGenCondOperatorComma);
2685   EXPECT_TOKEN(Tokens[12], tok::colon, TT_TableGenCondOperatorColon);
2686   // DAGArg values with operator identifier
2687   Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2688   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2689   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
2690   EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
2691   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // $src1
2692   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
2693   EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
2694   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2695   // List literal
2696   Tokens = AnnotateValue("[1, 2, 3]");
2697   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2698   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_TableGenListOpener);
2699   EXPECT_TOKEN(Tokens[6], tok::r_square, TT_TableGenListCloser);
2700   // Suffixes of values
2701   Tokens = AnnotateValue("valid.field");
2702   ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2703   EXPECT_TOKEN(Tokens[1], tok::period, TT_TableGenValueSuffix);
2704   // Code
2705   Tokens = AnnotateValue("[{ code is multiline string }]");
2706   ASSERT_EQ(Tokens.size(), 1u) << Tokens;
2707   EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2708 
2709   // The definition
2710   Tokens = annotate("def Def : Parent<Child> {}", Style);
2711   ASSERT_EQ(Tokens.size(), 10u) << Tokens; // This contains eof.
2712   // We use inheritance colon and function brace. They are enough.
2713   EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2714   EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
2715   EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
2716   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
2717 
2718   // DAGArg breaking options. They use different token types depending on what
2719   // is specified.
2720   Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakElements;
2721 
2722   // When TableGenBreakInsideDAGArg is DAS_BreakElements and
2723   // TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
2724   // elements to have line break.
2725   Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2726   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2727   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2728   EXPECT_TOKEN(Tokens[1], tok::identifier,
2729                TT_TableGenDAGArgOperatorID); // ins
2730   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2731   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2732 
2733   Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2734   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2735   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2736   EXPECT_TOKEN(Tokens[1], tok::identifier,
2737                TT_TableGenDAGArgOperatorID); // other
2738   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2739   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2740 
2741   // For non-identifier operators, breaks after the operator.
2742   Tokens = AnnotateValue("(!cast<Type>(\"Name\") type1:$src1, type2:$src2)");
2743   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2744   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2745   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_TableGenDAGArgOperatorToBreak);
2746   EXPECT_TOKEN(Tokens[11], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2747   EXPECT_TOKEN(Tokens[15], tok::r_paren, TT_TableGenDAGArgCloser);
2748 
2749   Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll;
2750 
2751   // When TableGenBreakInsideDAGArg is DAS_BreakAll and
2752   // TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
2753   // to have line break inside it.
2754   Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2755   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2756   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2757   EXPECT_TOKEN(Tokens[1], tok::identifier,
2758                TT_TableGenDAGArgOperatorToBreak); // ins
2759   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2760   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2761 
2762   Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2763   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2764   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2765   EXPECT_TOKEN(Tokens[1], tok::identifier,
2766                TT_TableGenDAGArgOperatorToBreak); // other
2767   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2768   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2769 
2770   // If TableGenBreakingDAGArgOperators is specified, it is limited to the
2771   // specified operators.
2772   Style.TableGenBreakingDAGArgOperators = {"ins", "outs"};
2773   Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2774   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2775   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2776   EXPECT_TOKEN(Tokens[1], tok::identifier,
2777                TT_TableGenDAGArgOperatorToBreak); // ins
2778   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2779   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2780 
2781   Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2782   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2783   EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
2784   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
2785   EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
2786   EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2787 
2788   // If TableGenBreakingDAGArgOperators is enabled, it uses
2789   // TT_TableGenDAGArgListColonToAlign to annotate the colon to align.
2790   Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true;
2791   Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2792   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2793   EXPECT_TOKEN(Tokens[1], tok::identifier,
2794                TT_TableGenDAGArgOperatorToBreak); // ins
2795   EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign);
2796   EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign);
2797 
2798   Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2799   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2800   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
2801   EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
2802   EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
2803 }
2804 
2805 TEST_F(TokenAnnotatorTest, UnderstandConstructors) {
2806   auto Tokens = annotate("Class::Class() : BaseClass(), Member() {}");
2807 
2808   // The TT_Unknown is clearly not binding for the future, please adapt if those
2809   // tokens get annotated.
2810   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2811   EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
2812   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
2813   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);
2814   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2815   EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
2816   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
2817   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);
2818   EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);
2819   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2820   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
2821 
2822   Tokens = annotate("Class::Class() : BaseClass{}, Member{} {}");
2823   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2824   EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
2825   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
2826   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);
2827   EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);
2828   EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
2829   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
2830   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);
2831   EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);
2832   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2833   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
2834 
2835   Tokens = annotate("class Class {\n"
2836                     "  Class() : BaseClass() {\n"
2837                     "#if 0\n"
2838                     "    // comment\n"
2839                     "#endif\n"
2840                     "  }\n"
2841                     "  Class f();\n"
2842                     "}");
2843   ASSERT_EQ(Tokens.size(), 25u) << Tokens;
2844   EXPECT_TOKEN(Tokens[6], tok::colon, TT_CtorInitializerColon);
2845   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_FunctionLBrace);
2846   EXPECT_BRACE_KIND(Tokens[10], BK_Block);
2847 }
2848 
2849 TEST_F(TokenAnnotatorTest, UnderstandsConditionParens) {
2850   auto Tokens = annotate("if (x) {}");
2851   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2852   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2853   Tokens = annotate("if constexpr (x) {}");
2854   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2855   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2856   Tokens = annotate("if CONSTEXPR (x) {}");
2857   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2858   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2859   Tokens = annotate("if (x) {} else if (x) {}");
2860   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2861   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2862   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_ConditionLParen);
2863 }
2864 
2865 TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {
2866   FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
2867 
2868   auto Tokens = annotate("int? a;", Style);
2869   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2870   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2871 
2872   Tokens = annotate("int? a = 1;", Style);
2873   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2874   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2875 
2876   Tokens = annotate("int?)", Style);
2877   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
2878   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2879 
2880   Tokens = annotate("int?>", Style);
2881   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
2882   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2883 
2884   Tokens = annotate("cond? id : id2", Style);
2885   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2886   EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
2887 
2888   Tokens = annotate("cond ? cond2 ? : id1 : id2", Style);
2889   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2890   EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
2891 }
2892 
2893 TEST_F(TokenAnnotatorTest, UnderstandsLabels) {
2894   auto Tokens = annotate("{ x: break; }");
2895   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2896   EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
2897 
2898   Tokens = annotate("{ case x: break; }");
2899   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2900   EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
2901 
2902   Tokens = annotate("{ x: { break; } }");
2903   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2904   EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
2905 
2906   Tokens = annotate("{ case x: { break; } }");
2907   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2908   EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
2909 
2910   Tokens = annotate("#define FOO label:");
2911   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2912   EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);
2913 
2914   Tokens = annotate("#define FOO \\\n"
2915                     "label: \\\n"
2916                     "  break;");
2917   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2918   EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);
2919 }
2920 
2921 TEST_F(TokenAnnotatorTest, UnderstandsNestedBlocks) {
2922   // The closing braces are not annotated. It doesn't seem to cause a problem.
2923   // So we only test for the opening braces.
2924   auto Tokens = annotate("{\n"
2925                          "  {\n"
2926                          "    { int a = 0; }\n"
2927                          "  }\n"
2928                          "  {}\n"
2929                          "}");
2930   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2931   EXPECT_BRACE_KIND(Tokens[0], BK_Block);
2932   EXPECT_BRACE_KIND(Tokens[1], BK_Block);
2933   EXPECT_BRACE_KIND(Tokens[2], BK_Block);
2934   EXPECT_BRACE_KIND(Tokens[10], BK_Block);
2935 }
2936 
2937 TEST_F(TokenAnnotatorTest, UnderstandDesignatedInitializers) {
2938   auto Tokens = annotate("SomeStruct { .a = 1 };");
2939   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2940   EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2941   EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
2942 
2943   Tokens = annotate("SomeStruct { .a = 1, .b = 2 };");
2944   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2945   EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2946   EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
2947   EXPECT_TOKEN(Tokens[7], tok::period, TT_DesignatedInitializerPeriod);
2948 
2949   Tokens = annotate("SomeStruct {\n"
2950                     "#ifdef FOO\n"
2951                     "  .a = 1,\n"
2952                     "#endif\n"
2953                     "  .b = 2\n"
2954                     "};");
2955   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2956   EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2957   EXPECT_TOKEN(Tokens[5], tok::period, TT_DesignatedInitializerPeriod);
2958   EXPECT_TOKEN(Tokens[12], tok::period, TT_DesignatedInitializerPeriod);
2959 
2960   Tokens = annotate("SomeStruct {\n"
2961                     "#if defined FOO\n"
2962                     "  .a = 1,\n"
2963                     "#endif\n"
2964                     "  .b = 2\n"
2965                     "};");
2966   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2967   EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2968   EXPECT_TOKEN(Tokens[6], tok::period, TT_DesignatedInitializerPeriod);
2969   EXPECT_TOKEN(Tokens[13], tok::period, TT_DesignatedInitializerPeriod);
2970 
2971   Tokens = annotate("Foo foo[] = {[0]{}};");
2972   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2973   EXPECT_TOKEN(Tokens[6], tok::l_square, TT_DesignatedInitializerLSquare);
2974   EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2975 }
2976 
2977 TEST_F(TokenAnnotatorTest, UnderstandsJavaScript) {
2978   auto Annotate = [this](StringRef Code) {
2979     return annotate(Code, getLLVMStyle(FormatStyle::LK_JavaScript));
2980   };
2981 
2982   // Dictionary.
2983   auto Tokens = Annotate("var x = {'x' : 1, 'y' : 2};");
2984   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2985   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_DictLiteral);
2986   EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_SelectorName);
2987   EXPECT_TOKEN(Tokens[5], tok::colon, TT_DictLiteral);
2988   EXPECT_TOKEN(Tokens[8], tok::string_literal, TT_SelectorName);
2989   EXPECT_TOKEN(Tokens[9], tok::colon, TT_DictLiteral);
2990   // Change when we need to annotate these.
2991   EXPECT_BRACE_KIND(Tokens[3], BK_Unknown);
2992   EXPECT_BRACE_KIND(Tokens[11], BK_Unknown);
2993   EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_Unknown);
2994 }
2995 
2996 TEST_F(TokenAnnotatorTest, UnderstandsAttributes) {
2997   auto Tokens = annotate("bool foo __attribute__((unused));");
2998   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2999   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
3000   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3001   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_Unknown);
3002   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_Unknown);
3003   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
3004 
3005   Tokens = annotate("bool foo __declspec(dllimport);");
3006   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3007   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3008   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
3009 
3010   Tokens = annotate("bool __attribute__((unused)) foo;");
3011   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3012   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
3013   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3014   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
3015   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
3016   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_StartOfName);
3017 
3018   Tokens = annotate("void __attribute__((x)) Foo();");
3019   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3020   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
3021   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3022   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
3023   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
3024   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_FunctionDeclarationName);
3025   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_FunctionDeclarationLParen);
3026 
3027   FormatStyle Style = getLLVMStyle();
3028   Style.AttributeMacros.push_back("FOO");
3029   Tokens = annotate("bool foo FOO(unused);", Style);
3030   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3031   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_AttributeMacro);
3032   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3033   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
3034 }
3035 
3036 TEST_F(TokenAnnotatorTest, UnderstandsControlStatements) {
3037   auto Tokens = annotate("while (true) {}");
3038   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3039   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3040   EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3041 
3042   Tokens = annotate("for (;;) {}");
3043   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3044   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_ControlStatementLBrace);
3045   EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_ControlStatementRBrace);
3046 
3047   Tokens = annotate("do {} while (true);");
3048   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3049   EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_ControlStatementLBrace);
3050   EXPECT_TOKEN(Tokens[2], tok::r_brace, TT_ControlStatementRBrace);
3051 
3052   Tokens = annotate("if (true) {} else if (false) {} else {}");
3053   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
3054   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3055   EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3056   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_ControlStatementLBrace);
3057   EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_ControlStatementRBrace);
3058   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_ElseLBrace);
3059   EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_ElseRBrace);
3060 
3061   Tokens = annotate("switch (foo) {}");
3062   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3063   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3064   EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3065 }
3066 
3067 TEST_F(TokenAnnotatorTest, UnderstandsDoWhile) {
3068   auto Tokens = annotate("do { ++i; } while ( i > 5 );");
3069   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3070   EXPECT_TOKEN(Tokens[6], tok::kw_while, TT_DoWhile);
3071 
3072   Tokens = annotate("do ++i; while ( i > 5 );");
3073   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3074   EXPECT_TOKEN(Tokens[4], tok::kw_while, TT_DoWhile);
3075 }
3076 
3077 TEST_F(TokenAnnotatorTest, StartOfName) {
3078   auto Tokens = annotate("#pragma clang diagnostic push");
3079   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3080   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3081   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3082   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
3083 
3084   Tokens = annotate("#pragma clang diagnostic ignored \"-Wzero-length-array\"");
3085   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3086   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3087   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3088   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
3089 
3090   Tokens = annotate("#define FOO Foo foo");
3091   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3092   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3093   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3094   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_StartOfName);
3095 
3096   Tokens = annotate("@interface NSCoder (TestCoder)");
3097   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3098   EXPECT_TOKEN(Tokens[0], tok::at, TT_ObjCDecl);
3099   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_StartOfName);
3100 
3101   auto Style = getLLVMStyle();
3102   Style.StatementAttributeLikeMacros.push_back("emit");
3103   Tokens = annotate("emit foo = 0;", Style);
3104   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3105   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
3106   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3107 }
3108 
3109 TEST_F(TokenAnnotatorTest, BraceKind) {
3110   auto Tokens = annotate("void f() {};");
3111   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3112   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3113   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3114   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3115   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3116   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3117 
3118   Tokens = annotate("class Foo<int> f() {}");
3119   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3120   EXPECT_TOKEN(Tokens[5], tok::identifier, TT_FunctionDeclarationName);
3121   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
3122   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
3123   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3124   EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3125 
3126   Tokens = annotate("template <typename T> class Foo<T> f() {}");
3127   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3128   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
3129   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3130   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3131   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3132   EXPECT_BRACE_KIND(Tokens[14], BK_Block);
3133 
3134   Tokens = annotate("void f() override {};");
3135   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3136   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3137   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3138   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
3139   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3140   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3141 
3142   Tokens = annotate("void f() noexcept(false) {};");
3143   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3144   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3145   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3146   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
3147   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3148   EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3149 
3150   Tokens = annotate("auto f() -> void {};");
3151   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3152   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3153   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3154   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3155   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3156   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3157 
3158   Tokens = annotate("void f() { /**/ };");
3159   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3160   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3161   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3162   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3163   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3164   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3165 
3166   Tokens = annotate("void f() { //\n"
3167                     "};");
3168   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3169   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3170   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3171   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3172   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3173   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3174 
3175   Tokens = annotate("void f() {\n"
3176                     "  //\n"
3177                     "};");
3178   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3179   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3180   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3181   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3182   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3183   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3184 
3185   Tokens = annotate("struct Foo {\n"
3186                     "  Foo() {};\n"
3187                     "  ~Foo() {};\n"
3188                     "};");
3189   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
3190   EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
3191   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
3192   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3193   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3194   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3195   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
3196   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3197   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3198   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3199   EXPECT_BRACE_KIND(Tokens[14], BK_Block);
3200 
3201   Tokens = annotate("{\n"
3202                     "  char *a[] = {\n"
3203                     "      /* abc */ \"abc\",\n"
3204                     "#if FOO\n"
3205                     "      /* xyz */ \"xyz\",\n"
3206                     "#endif\n"
3207                     "      /* last */ \"last\"};\n"
3208                     "}");
3209   ASSERT_EQ(Tokens.size(), 25u) << Tokens;
3210   EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3211   EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3212   EXPECT_BRACE_KIND(Tokens[21], BK_BracedInit);
3213 
3214   Tokens =
3215       annotate("#define SCOP_STAT(NAME, DESC) \\\n"
3216                "  {\"polly\", #NAME, \"Number of rejected regions: \" DESC}");
3217   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
3218   EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
3219   EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit);
3220 
3221   Tokens = annotate("struct {};");
3222   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3223   EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3224   EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3225 
3226   Tokens = annotate("struct : Base {};");
3227   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3228   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3229   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3230 
3231   Tokens = annotate("struct Foo {};");
3232   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3233   EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3234   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3235 
3236   Tokens = annotate("struct ::Foo {};");
3237   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3238   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3239   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3240 
3241   Tokens = annotate("struct NS::Foo {};");
3242   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3243   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3244   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3245 
3246   Tokens = annotate("struct Foo<int> {};");
3247   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3248   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3249   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3250 
3251   Tokens = annotate("struct Foo<int>::Bar {};");
3252   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3253   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3254   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3255 
3256   Tokens = annotate("struct Foo<int> : Base {};");
3257   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3258   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3259   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3260 
3261   Tokens = annotate("struct Foo final {};");
3262   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3263   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3264   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3265 
3266   Tokens = annotate("struct [[foo]] [[bar]] Foo final : Base1, Base2 {};");
3267   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
3268   EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3269   EXPECT_BRACE_KIND(Tokens[18], BK_Block);
3270 
3271   Tokens = annotate("struct Foo x{};");
3272   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3273   EXPECT_BRACE_KIND(Tokens[3], BK_BracedInit);
3274   EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3275 
3276   Tokens = annotate("struct ::Foo x{};");
3277   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3278   EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3279   EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3280 
3281   Tokens = annotate("struct NS::Foo x{};");
3282   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3283   EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3284   EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3285 
3286   Tokens = annotate("struct Foo<int> x{};");
3287   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3288   EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3289   EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3290 
3291   Tokens = annotate("#ifdef DEBUG_ENABLED\n"
3292                     "#else\n"
3293                     "#endif\n"
3294                     "class RenderingServer : Object {\n"
3295                     "#ifndef DISABLE_DEPRECATED\n"
3296                     "  enum Features {\n"
3297                     "    FEATURE_SHADERS,\n"
3298                     "    FEATURE_MULTITHREADED,\n"
3299                     "  };\n"
3300                     "#endif\n"
3301                     "};");
3302   ASSERT_EQ(Tokens.size(), 29u) << Tokens;
3303   EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3304   EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3305   EXPECT_BRACE_KIND(Tokens[22], BK_Block);
3306   EXPECT_BRACE_KIND(Tokens[26], BK_Block);
3307 
3308   Tokens = annotate("{\n"
3309                     "#define M(x) \\\n"
3310                     "  return {#x};\n"
3311                     "}");
3312   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
3313   EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3314   EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3315   EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
3316   EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
3317   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3318 
3319   Tokens = annotate("a = class extends goog.a {};",
3320                     getGoogleStyle(FormatStyle::LK_JavaScript));
3321   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3322   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_ClassLBrace);
3323   EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3324   EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_ClassRBrace);
3325   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3326 
3327   Tokens = annotate("a = class Foo extends goog.a {};",
3328                     getGoogleStyle(FormatStyle::LK_JavaScript));
3329   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3330   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_ClassLBrace);
3331   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3332   EXPECT_TOKEN(Tokens[9], tok::r_brace, TT_ClassRBrace);
3333   EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3334 
3335   Tokens = annotate("#define FOO(X) \\\n"
3336                     "  struct X##_tag_ {};");
3337   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3338   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_StructLBrace);
3339   EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3340   EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_StructRBrace);
3341   EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3342 
3343   Tokens = annotate("#define MACRO            \\\n"
3344                     "  struct hash<type> {    \\\n"
3345                     "    void f() { return; } \\\n"
3346                     "  };");
3347   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3348   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
3349   EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3350   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
3351   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3352   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3353   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3354   EXPECT_BRACE_KIND(Tokens[16], BK_Block);
3355   EXPECT_TOKEN(Tokens[17], tok::r_brace, TT_StructRBrace);
3356   EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3357 
3358   Tokens = annotate("#define MEMBER(NAME) NAME{\"\"}");
3359   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3360   EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3361   EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
3362 }
3363 
3364 TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) {
3365   auto Tokens = annotate("auto foo() -> enum En {}");
3366   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3367   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
3368 }
3369 
3370 TEST_F(TokenAnnotatorTest, BlockLBrace) {
3371   auto Tokens = annotate("{\n"
3372                          "  {\n"
3373                          "    foo();\n"
3374                          "  }\n"
3375                          "}");
3376   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3377   EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3378   EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3379   EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
3380   EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3381 
3382   Tokens = annotate("void bar() {\n"
3383                     "  {\n"
3384                     "    foo();\n"
3385                     "  }\n"
3386                     "}");
3387   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3388   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3389   EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3390   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_BlockLBrace);
3391   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3392 
3393   Tokens = annotate("[foo bar:{{0, 1}} baz:baz];",
3394                     getLLVMStyle(FormatStyle::LK_ObjC));
3395   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
3396   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not TT_BlockLBrace.
3397   EXPECT_BRACE_KIND(Tokens[4], BK_Unknown);          // Not BK_Block.
3398   EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3399   EXPECT_BRACE_KIND(Tokens[9], BK_Unknown);  // Not BK_Block.
3400   EXPECT_BRACE_KIND(Tokens[10], BK_Unknown); // Not BK_Block.
3401 }
3402 
3403 TEST_F(TokenAnnotatorTest, SwitchExpression) {
3404   auto Style = getLLVMStyle(FormatStyle::LK_Java);
3405   auto Tokens = annotate("i = switch (day) {\n"
3406                          "  case THURSDAY, SATURDAY -> 8;\n"
3407                          "  case WEDNESDAY -> 9;\n"
3408                          "  default -> 1;\n"
3409                          "};",
3410                          Style);
3411   ASSERT_EQ(Tokens.size(), 26u) << Tokens;
3412   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_SwitchExpressionLBrace);
3413   EXPECT_TOKEN(Tokens[7], tok::kw_case, TT_SwitchExpressionLabel);
3414   EXPECT_TOKEN(Tokens[11], tok::arrow, TT_CaseLabelArrow);
3415   EXPECT_TOKEN(Tokens[14], tok::kw_case, TT_SwitchExpressionLabel);
3416   EXPECT_TOKEN(Tokens[16], tok::arrow, TT_CaseLabelArrow);
3417   EXPECT_TOKEN(Tokens[19], tok::kw_default, TT_SwitchExpressionLabel);
3418   EXPECT_TOKEN(Tokens[20], tok::arrow, TT_CaseLabelArrow);
3419 }
3420 
3421 TEST_F(TokenAnnotatorTest, CppAltOperatorKeywords) {
3422   auto Tokens = annotate("a = b and c;");
3423   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3424   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
3425 
3426   Tokens = annotate("a = b and_eq c;");
3427   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3428   EXPECT_TOKEN(Tokens[3], tok::ampequal, TT_BinaryOperator);
3429 
3430   Tokens = annotate("a = b bitand c;");
3431   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3432   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
3433 
3434   Tokens = annotate("a = b bitor c;");
3435   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3436   EXPECT_TOKEN(Tokens[3], tok::pipe, TT_BinaryOperator);
3437 
3438   Tokens = annotate("a = b compl c;");
3439   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3440   EXPECT_TOKEN(Tokens[3], tok::tilde, TT_UnaryOperator);
3441 
3442   Tokens = annotate("a = b not c;");
3443   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3444   EXPECT_TOKEN(Tokens[3], tok::exclaim, TT_UnaryOperator);
3445 
3446   Tokens = annotate("a = b not_eq c;");
3447   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3448   EXPECT_TOKEN(Tokens[3], tok::exclaimequal, TT_BinaryOperator);
3449 
3450   Tokens = annotate("a = b or c;");
3451   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3452   EXPECT_TOKEN(Tokens[3], tok::pipepipe, TT_BinaryOperator);
3453 
3454   Tokens = annotate("a = b or_eq c;");
3455   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3456   EXPECT_TOKEN(Tokens[3], tok::pipeequal, TT_BinaryOperator);
3457 
3458   Tokens = annotate("a = b xor c;");
3459   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3460   EXPECT_TOKEN(Tokens[3], tok::caret, TT_BinaryOperator);
3461 
3462   Tokens = annotate("a = b xor_eq c;");
3463   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3464   EXPECT_TOKEN(Tokens[3], tok::caretequal, TT_BinaryOperator);
3465 
3466   Tokens = annotate("xor = foo;");
3467   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3468   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
3469 
3470   Tokens = annotate("int xor = foo;");
3471   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3472   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
3473 }
3474 
3475 TEST_F(TokenAnnotatorTest, FunctionTryBlock) {
3476   auto Tokens =
3477       annotate("Foo::Foo(int x, int y) try\n"
3478                "    : foo{[] -> std::string { return {}; }(), x}, bar{y} {\n"
3479                "} catch (...) {\n"
3480                "}");
3481   ASSERT_EQ(Tokens.size(), 45u) << Tokens;
3482   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
3483   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
3484   EXPECT_TOKEN(Tokens[11], tok::colon, TT_CtorInitializerColon);
3485   EXPECT_TOKEN(Tokens[14], tok::l_square, TT_LambdaLSquare);
3486   EXPECT_TOKEN(Tokens[16], tok::arrow, TT_LambdaArrow);
3487   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
3488   EXPECT_TOKEN(Tokens[31], tok::comma, TT_CtorInitializerComma);
3489   EXPECT_TOKEN(Tokens[36], tok::l_brace, TT_FunctionLBrace);
3490 }
3491 
3492 TEST_F(TokenAnnotatorTest, TypenameMacro) {
3493   auto Style = getLLVMStyle();
3494   Style.TypenameMacros.push_back("STRUCT");
3495 
3496   auto Tokens = annotate("STRUCT(T, B) { int i; };", Style);
3497   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3498   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TypenameMacro);
3499   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
3500   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_TypeDeclarationParen);
3501   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_Unknown);
3502 }
3503 
3504 TEST_F(TokenAnnotatorTest, GNULanguageStandard) {
3505   auto Style = getGNUStyle();
3506   EXPECT_EQ(Style.Standard, FormatStyle::LS_Latest);
3507 
3508   auto Tokens = annotate("return 1 <=> 2;", Style);
3509   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3510   EXPECT_TOKEN(Tokens[2], tok::spaceship, TT_BinaryOperator);
3511 }
3512 
3513 TEST_F(TokenAnnotatorTest, SplitPenalty) {
3514   auto Style = getLLVMStyle();
3515   Style.ColumnLimit = 20;
3516 
3517   auto Tokens = annotate("class foo {\n"
3518                          "  auto bar()\n"
3519                          "      -> bool;\n"
3520                          "};",
3521                          Style);
3522   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3523   EXPECT_TOKEN(Tokens[7], tok::arrow, TT_TrailingReturnArrow);
3524   EXPECT_SPLIT_PENALTY(Tokens[7], 23u);
3525 }
3526 
3527 TEST_F(TokenAnnotatorTest, TemplateName) {
3528   constexpr StringRef Code{"return Foo < A || B > (C ^ D);"};
3529 
3530   auto Tokens = annotate(Code);
3531   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3532   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
3533   EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
3534 
3535   auto Style = getLLVMStyle();
3536   Style.TemplateNames.push_back("Foo");
3537 
3538   Tokens = annotate(Code, Style);
3539   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_TemplateName);
3540   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
3541   EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
3542 }
3543 
3544 TEST_F(TokenAnnotatorTest, TemplateInstantiation) {
3545   auto Tokens = annotate("return FixedInt<N | M>();");
3546   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3547   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
3548   EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
3549 
3550   Tokens = annotate("return std::conditional_t<T::value == U::value, T, U>{};");
3551   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
3552   EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
3553   EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
3554 }
3555 
3556 } // namespace
3557 } // namespace format
3558 } // namespace clang
3559