xref: /llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp (revision 15e14f129fe24db989ed91737e96505863e20c60)
1 //===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Format/Format.h"
10 
11 #include "FormatTestUtils.h"
12 #include "TestLexer.h"
13 #include "gtest/gtest.h"
14 
15 namespace clang {
16 namespace format {
17 
18 // Not really the equality, but everything we need.
19 static bool operator==(const FormatToken &LHS,
20                        const FormatToken &RHS) noexcept {
21   return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22          LHS.getType() == RHS.getType();
23 }
24 
25 namespace {
26 
27 class TokenAnnotatorTest : public ::testing::Test {
28 protected:
29   TokenList annotate(llvm::StringRef Code,
30                      const FormatStyle &Style = getLLVMStyle()) {
31     return TestLexer(Allocator, Buffers, Style).annotate(Code);
32   }
33   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35 };
36 
37 #define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \
38   EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39 #define EXPECT_TOKEN_TYPE(FormatTok, Type)                                     \
40   EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41 #define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec)                               \
42   EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)
43 #define EXPECT_BRACE_KIND(FormatTok, Kind)                                     \
44   EXPECT_EQ(FormatTok->getBlockKind(), Kind) << *(FormatTok)
45 #define EXPECT_TOKEN(FormatTok, Kind, Type)                                    \
46   do {                                                                         \
47     EXPECT_TOKEN_KIND(FormatTok, Kind);                                        \
48     EXPECT_TOKEN_TYPE(FormatTok, Type);                                        \
49   } while (false)
50 
51 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
52   auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};");
53   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
54   EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
55   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
56   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
57   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
58   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
59 
60   Tokens = annotate("auto x = [](const decltype(x) *ptr) {};");
61   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
62   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
63   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
64 
65   Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}");
66   EXPECT_EQ(Tokens.size(), 17u) << Tokens;
67   EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
68   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
69   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
70   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
71   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
72 
73   Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}");
74   EXPECT_EQ(Tokens.size(), 17u) << Tokens;
75   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
76   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
77 
78   Tokens = annotate("void f() {\n"
79                     "  while (p < a && *p == 'a')\n"
80                     "    p++;\n"
81                     "}");
82   EXPECT_EQ(Tokens.size(), 21u) << Tokens;
83   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
84   EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
85 
86   Tokens = annotate("case *x:");
87   EXPECT_EQ(Tokens.size(), 5u) << Tokens;
88   EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);
89   Tokens = annotate("case &x:");
90   EXPECT_EQ(Tokens.size(), 5u) << Tokens;
91   EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);
92 
93   Tokens = annotate("bool b = 3 == int{3} && true;\n");
94   EXPECT_EQ(Tokens.size(), 13u) << Tokens;
95   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
96 
97   Tokens = annotate("struct {\n"
98                     "} *ptr;");
99   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
100   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
101   Tokens = annotate("union {\n"
102                     "} *ptr;");
103   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
104   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
105   Tokens = annotate("class {\n"
106                     "} *ptr;");
107   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
108   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
109 
110   Tokens = annotate("struct {\n"
111                     "} &&ptr = {};");
112   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
113   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
114   Tokens = annotate("union {\n"
115                     "} &&ptr = {};");
116   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
117   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
118   Tokens = annotate("class {\n"
119                     "} &&ptr = {};");
120   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
121   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
122   Tokens = annotate("int i = int{42} * 2;");
123   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
124   EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);
125 
126   Tokens = annotate("delete[] *ptr;");
127   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
128   EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
129   Tokens = annotate("delete[] **ptr;");
130   EXPECT_EQ(Tokens.size(), 8u) << Tokens;
131   EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
132   EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);
133   Tokens = annotate("delete[] *(ptr);");
134   EXPECT_EQ(Tokens.size(), 9u) << Tokens;
135   EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
136 
137   Tokens = annotate("void f() { void (*fnptr)(char* foo); }");
138   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
139   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
140   // FIXME: The star of a function pointer probably makes more sense as
141   // TT_PointerOrReference.
142   EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
143   EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
144 
145   Tokens = annotate("void f() { void (*fnptr)(t* foo); }");
146   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
147   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
148   EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
149   EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
150 
151   Tokens = annotate("int f3() { return sizeof(Foo&); }");
152   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
153   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
154 
155   Tokens = annotate("int f4() { return sizeof(Foo&&); }");
156   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
157   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
158 
159   Tokens = annotate("void f5() { int f6(Foo&, Bar&); }");
160   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
161   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
162   EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
163 
164   Tokens = annotate("void f7() { int f8(Foo&&, Bar&&); }");
165   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
166   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
167   EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
168 
169   Tokens = annotate("Type1 &val1 = val2;");
170   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
171   EXPECT_TOKEN(Tokens[1], tok::amp, TT_PointerOrReference);
172 
173   Tokens = annotate("Type1 *val1 = &val2;");
174   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
175   EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
176   EXPECT_TOKEN(Tokens[4], tok::amp, TT_UnaryOperator);
177 
178   Tokens = annotate("val1 & val2;");
179   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
180   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
181 
182   Tokens = annotate("val1 & val2.member;");
183   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
184   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
185 
186   Tokens = annotate("val1 & val2.*member;");
187   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
188   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
189 
190   Tokens = annotate("val1.*member & val2;");
191   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
192   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
193 
194   Tokens = annotate("val1 & val2->*member;");
195   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
196   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
197 
198   Tokens = annotate("val1->member & val2;");
199   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
200   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
201 
202   Tokens = annotate("val1 & val2 & val3;");
203   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
204   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
205   EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
206 
207   Tokens = annotate("val1 & val2 // comment\n"
208                     "     & val3;");
209   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
210   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
211   EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
212 
213   Tokens =
214       annotate("val1 & val2.member & val3.member() & val4 & val5->member;");
215   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
216   EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
217   EXPECT_TOKEN(Tokens[5], tok::amp, TT_BinaryOperator);
218   EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
219   EXPECT_TOKEN(Tokens[13], tok::amp, TT_BinaryOperator);
220 
221   Tokens = annotate("class c {\n"
222                     "  void func(type &a) { a & member; }\n"
223                     "  anotherType &member;\n"
224                     "}");
225   ASSERT_EQ(Tokens.size(), 22u) << Tokens;
226   EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
227   EXPECT_TOKEN(Tokens[12], tok::amp, TT_BinaryOperator);
228   EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);
229 
230   Tokens = annotate("struct S {\n"
231                     "  auto Mem = C & D;\n"
232                     "}");
233   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
234   EXPECT_TOKEN(Tokens[7], tok::amp, TT_BinaryOperator);
235 
236   Tokens =
237       annotate("template <typename T> void swap() noexcept(Bar<T> && Foo<T>);");
238   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
239   EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
240 
241   Tokens = annotate("template <typename T> struct S {\n"
242                     "  explicit(Bar<T> && Foo<T>) S(const S &);\n"
243                     "};");
244   ASSERT_EQ(Tokens.size(), 30u) << Tokens;
245   EXPECT_TOKEN(Tokens[14], tok::ampamp, TT_BinaryOperator);
246 
247   Tokens = annotate("template <bool B = C && D> struct S {};");
248   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
249   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
250 
251   Tokens = annotate("template <typename T, bool B = C && D> struct S {};");
252   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
253   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
254 
255   Tokens = annotate("template <typename T, typename U = T&&> struct S {};");
256   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
257   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
258 
259   Tokens = annotate("template <typename T = int (*)(int)> struct S {};");
260   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
261   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
262   EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);
263 
264   Tokens = annotate("Foo<A && B> a = {};");
265   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
266   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
267 
268   Tokens = annotate("Foo<A &&> a = {};");
269   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
270   EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
271 
272   Tokens = annotate("template <enable_if_t<foo && !bar>* = nullptr> void f();");
273   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
274   EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_BinaryOperator);
275 }
276 
277 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
278   auto Tokens = annotate("x - 0");
279   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
280   EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);
281   Tokens = annotate("0 + 0");
282   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
283   EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);
284   Tokens = annotate("x + +0");
285   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
286   EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);
287   Tokens = annotate("x ? -0 : +0");
288   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
289   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
290   EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);
291   Tokens = annotate("(-0)");
292   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
293   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
294   Tokens = annotate("0, -0");
295   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
296   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
297   Tokens = annotate("for (; -1;) {\n}");
298   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
299   EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
300   Tokens = annotate("x = -1;");
301   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
302   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
303   Tokens = annotate("x[-1]");
304   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
305   EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
306   Tokens = annotate("x = {-1};");
307   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
308   EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
309   Tokens = annotate("case -x:");
310   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
311   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
312   Tokens = annotate("co_await -x;");
313   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
314   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
315   Tokens = annotate("co_return -x;");
316   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
317   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
318   Tokens = annotate("co_yield -x;");
319   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
320   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
321   Tokens = annotate("delete -x;");
322   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
323   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
324   Tokens = annotate("return -x;");
325   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
326   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
327   Tokens = annotate("throw -x;");
328   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
329   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
330   Tokens = annotate("sizeof -x");
331   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
332   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
333   Tokens = annotate("co_await +x;");
334   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
335   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
336   Tokens = annotate("co_return +x;");
337   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
338   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
339   Tokens = annotate("co_yield +x;");
340   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
341   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
342   Tokens = annotate("delete +x;");
343   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
344   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
345   Tokens = annotate("return +x;");
346   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
347   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
348   Tokens = annotate("throw +x;");
349   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
350   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
351   Tokens = annotate("sizeof +x");
352   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
353   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
354   Tokens = annotate("(int)-x");
355   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
356   EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
357   Tokens = annotate("(-x)");
358   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
359   EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
360   Tokens = annotate("!+x");
361   ASSERT_EQ(Tokens.size(), 4u) << Tokens;
362   EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);
363   EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
364 }
365 
366 TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
367   auto Tokens = annotate("class C {};");
368   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
369   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
370 
371   Tokens = annotate("const class C {} c;");
372   EXPECT_EQ(Tokens.size(), 8u) << Tokens;
373   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
374 
375   Tokens = annotate("const class {} c;");
376   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
377   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
378 
379   Tokens = annotate("class [[deprecated(\"\")]] C { int i; };");
380   EXPECT_EQ(Tokens.size(), 17u) << Tokens;
381   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
382 }
383 
384 TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
385   auto Tokens = annotate("struct S {};");
386   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
387   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
388 
389   Tokens = annotate("struct EXPORT_MACRO [[nodiscard]] C { int i; };");
390   EXPECT_EQ(Tokens.size(), 15u) << Tokens;
391   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
392 
393   Tokens = annotate("struct [[deprecated]] [[nodiscard]] C { int i; };");
394   EXPECT_EQ(Tokens.size(), 19u) << Tokens;
395   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
396 
397   Tokens = annotate("template <typename T> struct S<const T[N]> {};");
398   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
399   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
400   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
401   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
402   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
403 
404   Tokens = annotate("template <typename T> struct S<T const[N]> {};");
405   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
406   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
407   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
408   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
409   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
410 
411   Tokens = annotate("template <typename T, unsigned n> struct S<T const[n]> {\n"
412                     "  void f(T const (&a)[n]);\n"
413                     "};");
414   EXPECT_EQ(Tokens.size(), 35u) << Tokens;
415   EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);
416   EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);
417   EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
418   EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_StructLBrace);
419   EXPECT_TOKEN(Tokens[23], tok::l_paren, TT_FunctionTypeLParen);
420   EXPECT_TOKEN(Tokens[24], tok::amp, TT_UnaryOperator);
421   EXPECT_TOKEN(Tokens[27], tok::l_square, TT_ArraySubscriptLSquare);
422 }
423 
424 TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
425   auto Tokens = annotate("union U {};");
426   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
427   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
428 
429   Tokens = annotate("union U { void f() { return; } };");
430   EXPECT_EQ(Tokens.size(), 14u) << Tokens;
431   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
432   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
433 }
434 
435 TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
436   auto Tokens = annotate("enum E {};");
437   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
438   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
439 }
440 
441 TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
442   auto Tokens = annotate("auto operator<=>(const T &) const & = default;");
443   EXPECT_EQ(Tokens.size(), 14u) << Tokens;
444   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
445 
446   Tokens = annotate("template <typename T> void F(T) && = delete;");
447   EXPECT_EQ(Tokens.size(), 15u) << Tokens;
448   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
449 }
450 
451 TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
452   auto Tokens =
453       annotate("inline bool var = is_integral_v<int> && is_signed_v<int>;");
454   EXPECT_EQ(Tokens.size(), 15u) << Tokens;
455   EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
456 }
457 
458 TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
459   auto Tokens =
460       annotate("template <typename T> "
461                "inline bool var = is_integral_v<int> && is_signed_v<int>;");
462   EXPECT_EQ(Tokens.size(), 20u) << Tokens;
463   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
464 }
465 
466 TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {
467   auto Tokens =
468       annotate("#define FOO(typeName) \\\n"
469                "  { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");
470   ASSERT_EQ(Tokens.size(), 27u) << Tokens;
471   EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
472   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
473   EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);
474   EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);
475 }
476 
477 TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {
478   auto Tokens = annotate("if (std::tuple_size_v<T> > 0)");
479   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
480   EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
481   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
482   EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
483 }
484 
485 TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {
486   auto Tokens = annotate("return a < b && c > d;");
487   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
488   EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
489   EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
490 
491   Tokens = annotate("a < 0 ? b : a > 0 ? c : d;");
492   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
493   EXPECT_TOKEN(Tokens[1], tok::less, TT_BinaryOperator);
494   EXPECT_TOKEN(Tokens[7], tok::greater, TT_BinaryOperator);
495 
496   Tokens = annotate("ratio{-1, 2} < ratio{-1, 3} == -1 / 3 > -1 / 2;");
497   ASSERT_EQ(Tokens.size(), 27u) << Tokens;
498   EXPECT_TOKEN(Tokens[7], tok::less, TT_BinaryOperator);
499   EXPECT_TOKEN(Tokens[20], tok::greater, TT_BinaryOperator);
500 }
501 
502 TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
503   FormatStyle Style = getLLVMStyle();
504   Style.WhitespaceSensitiveMacros.push_back("FOO");
505 
506   auto Tokens = annotate("FOO(1+2 )\n", Style);
507   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
508   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
509 
510   Tokens = annotate("FOO(a:b:c)\n", Style);
511   EXPECT_EQ(Tokens.size(), 9u) << Tokens;
512   EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
513 }
514 
515 TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
516   auto Tokens = annotate("delete (void *)p;");
517   EXPECT_EQ(Tokens.size(), 8u) << Tokens;
518   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
519 
520   Tokens = annotate("delete[] (void *)p;");
521   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
522   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
523 
524   Tokens = annotate("delete[] /*comment*/ (void *)p;");
525   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
526   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
527 
528   Tokens = annotate("delete[/*comment*/] (void *)p;");
529   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
530   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
531 
532   Tokens = annotate("delete/*comment*/[] (void *)p;");
533   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
534   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
535 }
536 
537 TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
538   auto Tokens = annotate("(void)p;");
539   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
540   EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
541 
542   Tokens = annotate("auto x = (Foo)p;");
543   EXPECT_EQ(Tokens.size(), 9u) << Tokens;
544   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
545 
546   Tokens = annotate("(std::vector<int>)p;");
547   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
548   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
549 
550   Tokens = annotate("return (Foo)p;");
551   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
552   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
553 
554   Tokens = annotate("throw (Foo)p;");
555   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
556   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
557 }
558 
559 TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {
560   auto Tokens = annotate("void f() throw(int);");
561   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
562   EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);
563 }
564 
565 TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
566   auto Tokens = annotate("void f() &;");
567   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
568   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
569 
570   Tokens = annotate("void operator=(T) &&;");
571   EXPECT_EQ(Tokens.size(), 9u) << Tokens;
572   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
573 
574   Tokens = annotate("template <typename T> void f() &;");
575   EXPECT_EQ(Tokens.size(), 12u) << Tokens;
576   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
577 
578   Tokens = annotate("template <typename T> void operator=(T) &;");
579   EXPECT_EQ(Tokens.size(), 14u) << Tokens;
580   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
581 }
582 
583 TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
584   auto Tokens = annotate("x.operator+()");
585   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
586   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
587   EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);
588   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
589   Tokens = annotate("x.operator=()");
590   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
591   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
592   EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);
593   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
594   Tokens = annotate("x.operator+=()");
595   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
596   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
597   EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);
598   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
599   Tokens = annotate("x.operator,()");
600   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
601   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
602   EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);
603   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
604   Tokens = annotate("x.operator()()");
605   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
606   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
607   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);
608   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);
609   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
610   Tokens = annotate("x.operator[]()");
611   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
612   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
613   // EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);
614   // EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);
615   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
616   Tokens = annotate("x.operator\"\"_a()");
617   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
618   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
619   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
620   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
621   Tokens = annotate("x.operator\"\" _a()");
622   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
623   // FIXME
624   // EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
625   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
626   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
627   Tokens = annotate("x.operator\"\"if()");
628   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
629   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
630   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
631   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
632   Tokens = annotate("x.operator\"\"s()");
633   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
634   EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
635   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
636   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
637   Tokens = annotate("x.operator\"\" s()");
638   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
639   // FIXME
640   // EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
641   EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
642   EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
643 
644   Tokens = annotate("int operator+(int);");
645   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
646   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
647   EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
648   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
649   Tokens = annotate("auto operator=(T&) {}");
650   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
651   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
652   EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);
653   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
654   Tokens = annotate("auto operator()() {}");
655   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
656   EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
657   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);
658   EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);
659   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
660 }
661 
662 TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {
663   struct {
664     const char *Text;
665     tok::TokenKind Kind;
666   } Operators[] = {{"+", tok::plus},
667                    {"-", tok::minus},
668                    // FIXME:
669                    // {"*", tok::star},
670                    {"/", tok::slash},
671                    {"%", tok::percent},
672                    {"^", tok::caret},
673                    // FIXME:
674                    // {"&", tok::amp},
675                    {"|", tok::pipe},
676                    {"~", tok::tilde},
677                    {"!", tok::exclaim},
678                    {"=", tok::equal},
679                    // FIXME:
680                    // {"<", tok::less},
681                    {">", tok::greater},
682                    {"+=", tok::plusequal},
683                    {"-=", tok::minusequal},
684                    {"*=", tok::starequal},
685                    {"/=", tok::slashequal},
686                    {"%=", tok::percentequal},
687                    {"^=", tok::caretequal},
688                    {"&=", tok::ampequal},
689                    {"|=", tok::pipeequal},
690                    {"<<", tok::lessless},
691                    {">>", tok::greatergreater},
692                    {">>=", tok::greatergreaterequal},
693                    {"<<=", tok::lesslessequal},
694                    {"==", tok::equalequal},
695                    {"!=", tok::exclaimequal},
696                    {"<=", tok::lessequal},
697                    {">=", tok::greaterequal},
698                    {"<=>", tok::spaceship},
699                    {"&&", tok::ampamp},
700                    {"||", tok::pipepipe},
701                    {"++", tok::plusplus},
702                    {"--", tok::minusminus},
703                    {",", tok::comma},
704                    {"->*", tok::arrowstar},
705                    {"->", tok::arrow}};
706 
707   for (const auto &Operator : Operators) {
708     std::string Input("C<&operator");
709     Input += Operator.Text;
710     Input += " > a;";
711     auto Tokens = annotate(std::string(Input));
712     ASSERT_EQ(Tokens.size(), 9u) << Tokens;
713     EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
714     EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator);
715     EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
716   }
717 
718   auto Tokens = annotate("C<&operator< <X>> lt;");
719   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
720   EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
721   EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator);
722   EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
723   EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
724   EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
725 }
726 
727 TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
728   auto Tokens = annotate("template <typename T>\n"
729                          "concept C = (Foo && Bar) && (Bar && Baz);");
730 
731   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
732   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
733   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
734   EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
735 
736   Tokens = annotate("template <typename T>\n"
737                     "concept C = Foo && !Bar;");
738 
739   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
740   EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
741   EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);
742 
743   Tokens = annotate("template <typename T>\n"
744                     "concept C = requires(T t) {\n"
745                     "  { t.foo() };\n"
746                     "} && Bar<T> && Baz<T>;");
747   ASSERT_EQ(Tokens.size(), 35u) << Tokens;
748   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
749   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
750   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
751   EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
752   EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
753 
754   Tokens = annotate("template<typename T>\n"
755                     "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
756                     "struct Foo;");
757   ASSERT_EQ(Tokens.size(), 36u) << Tokens;
758   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
759   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
760   EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
761   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
762   EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
763   EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
764   EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
765   EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
766   EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
767   EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
768 
769   Tokens =
770       annotate("template<typename T>\n"
771                "requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
772                "struct Foo;");
773   ASSERT_EQ(Tokens.size(), 38u) << Tokens;
774   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
775   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
776   EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
777   EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
778   EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
779   EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
780   EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
781   EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
782   EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
783   EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
784   EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
785 
786   Tokens = annotate("template <typename T>\n"
787                     "void foo(T) noexcept requires Bar<T>;");
788   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
789   EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
790 
791   Tokens = annotate("template <typename T>\n"
792                     "requires Bar<T> || Baz<T>\n"
793                     "auto foo(T) -> int;");
794   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
795   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
796   EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);
797   EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
798   EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);
799 
800   Tokens = annotate("template <typename T>\n"
801                     "requires Bar<T>\n"
802                     "bool foo(T) { return false; }");
803   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
804   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
805   EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
806   EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
807 
808   Tokens = annotate("template <typename T>\n"
809                     "requires Bar<T>\n"
810                     "decltype(auto) foo(T) { return false; }");
811   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
812   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
813   EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
814   EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);
815 
816   Tokens = annotate("template <typename T>\n"
817                     "struct S {\n"
818                     "  void foo() const requires Bar<T>;\n"
819                     "  void bar() const & requires Baz<T>;\n"
820                     "  void bar() && requires Baz2<T>;\n"
821                     "  void baz() const & noexcept requires Baz<T>;\n"
822                     "  void baz() && noexcept requires Baz2<T>;\n"
823                     "};\n"
824                     "\n"
825                     "void S::bar() const & requires Baz<T> { }");
826   ASSERT_EQ(Tokens.size(), 85u) << Tokens;
827   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
828   EXPECT_TOKEN(Tokens[24], tok::amp, TT_PointerOrReference);
829   EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
830   EXPECT_TOKEN(Tokens[35], tok::ampamp, TT_PointerOrReference);
831   EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
832   EXPECT_TOKEN(Tokens[47], tok::amp, TT_PointerOrReference);
833   EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
834   EXPECT_TOKEN(Tokens[59], tok::ampamp, TT_PointerOrReference);
835   EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
836   EXPECT_TOKEN(Tokens[76], tok::amp, TT_PointerOrReference);
837   EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
838 
839   Tokens = annotate("void Class::member() && requires(Constant) {}");
840   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
841   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
842 
843   Tokens = annotate("void Class::member() && requires(Constant<T>) {}");
844   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
845   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
846 
847   Tokens =
848       annotate("void Class::member() && requires(Namespace::Constant<T>) {}");
849   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
850   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
851 
852   Tokens = annotate("void Class::member() && requires(typename "
853                     "Namespace::Outer<T>::Inner::Constant) {}");
854   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
855   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
856 
857   Tokens = annotate("struct [[nodiscard]] zero_t {\n"
858                     "  template<class T>\n"
859                     "    requires requires { number_zero_v<T>; }\n"
860                     "  [[nodiscard]] constexpr operator T() const { "
861                     "return number_zero_v<T>; }\n"
862                     "};");
863   ASSERT_EQ(Tokens.size(), 44u) << Tokens;
864   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
865   EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
866   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
867   EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
868   EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
869   EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
870 
871   Tokens =
872       annotate("template <class A, class B> concept C ="
873                "std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
874   ASSERT_EQ(Tokens.size(), 31u) << Tokens;
875   EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
876   EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
877   EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
878   EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);
879   EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);
880   EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);
881   EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);
882 
883   Tokens = annotate("auto bar() -> int requires(is_integral_v<T>) {}");
884   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
885   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
886 
887   Tokens = annotate("auto bar() -> void requires(is_integral_v<T>) {}");
888   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
889   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
890 
891   Tokens = annotate("auto bar() -> MyType requires(is_integral_v<T>) {}");
892   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
893   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
894 
895   Tokens =
896       annotate("auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");
897   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
898   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
899 
900   Tokens =
901       annotate("auto bar() -> qualified::type requires(is_integral_v<T>) {}");
902   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
903   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
904 
905   Tokens =
906       annotate("auto bar() -> Template<type> requires(is_integral_v<T>) {}");
907   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
908   EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);
909 
910   Tokens = annotate("void foo() requires((A<T>) && C) {}");
911   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
912   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
913   EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
914 
915   Tokens = annotate("void foo() requires(((A<T>) && C)) {}");
916   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
917   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
918   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
919 
920   Tokens = annotate("void foo() requires([](T&&){}(t)) {}");
921   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
922   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
923   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
924 
925   Tokens = annotate("void foo() requires([](T&& u){}(t)) {}");
926   ASSERT_EQ(Tokens.size(), 22u) << Tokens;
927   EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
928   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
929 }
930 
931 TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
932   auto Tokens = annotate("bool b = requires(int i) { i + 5; };");
933   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
934   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
935   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
936   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
937 
938   Tokens = annotate("if (requires(int i) { i + 5; }) return;");
939   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
940   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
941   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
942   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
943 
944   Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;");
945   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
946   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
947   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
948   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
949 
950   Tokens = annotate("foo(requires(const T t) {});");
951   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
952   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
953   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
954   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
955 
956   Tokens = annotate("foo(requires(const int t) {});");
957   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
958   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
959   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
960   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
961 
962   Tokens = annotate("foo(requires(const T t) {});");
963   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
964   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
965   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
966   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
967 
968   Tokens = annotate("foo(requires(int const* volatile t) {});");
969   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
970   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
971   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
972   EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
973   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
974 
975   Tokens = annotate("foo(requires(T const* volatile t) {});");
976   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
977   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
978   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
979   EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
980   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
981 
982   Tokens = annotate("foo(requires(T& t) {});");
983   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
984   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
985   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
986   EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);
987   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
988 
989   Tokens = annotate("foo(requires(T&& t) {});");
990   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
991   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
992   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
993   EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);
994   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
995 
996   Tokens = annotate("bool foo = requires(T& t) {};");
997   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
998   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
999   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1000   EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1001   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1002 
1003   Tokens = annotate("bool foo = requires(T&& t) {};");
1004   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1005   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1006   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1007   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
1008   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1009 
1010   Tokens =
1011       annotate("foo(requires(const typename Outer<T>::Inner * const t) {});");
1012   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1013   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1014   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1015   EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
1016   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
1017 
1018   Tokens = annotate("template <typename T>\n"
1019                     "concept C = requires(T T) {\n"
1020                     "  requires Bar<T> && Foo<T>;\n"
1021                     "};");
1022   ASSERT_EQ(Tokens.size(), 28u) << Tokens;
1023   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1024   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1025   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1026   EXPECT_TOKEN(Tokens[14], tok::kw_requires,
1027                TT_RequiresClauseInARequiresExpression);
1028 
1029   Tokens = annotate("template <typename T>\n"
1030                     "concept C = requires(T T) {\n"
1031                     "  { t.func() } -> std::same_as<int>;"
1032                     "  requires Bar<T> && Foo<T>;\n"
1033                     "};");
1034   ASSERT_EQ(Tokens.size(), 43u) << Tokens;
1035   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1036   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1037   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1038   EXPECT_TOKEN(Tokens[29], tok::kw_requires,
1039                TT_RequiresClauseInARequiresExpression);
1040 
1041   // Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
1042   Tokens = annotate("bool r10 = requires (struct new_struct { int x; } s) { "
1043                     "requires true; };");
1044   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1045   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1046   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1047   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
1048 }
1049 
1050 TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {
1051   // Everything after #pragma region should be ImplicitStringLiteral
1052   auto Tokens = annotate("#pragma region Foo(Bar: Hello)");
1053   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1054   EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);
1055   EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);
1056   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);
1057 
1058   // Make sure it's annotated correctly inside a function as well
1059   Tokens = annotate("void test(){\n#pragma region Foo(Bar: Hello)\n}");
1060   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1061   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);
1062   EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);
1063   EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);
1064 }
1065 
1066 TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
1067   const char *BaseCode = nullptr;
1068   const char *ConstrainedCode = nullptr;
1069   auto BaseTokenCount = 0u;
1070   auto RequiresTokenCount = 0u;
1071   auto PrefixTokenCount = 0u;
1072 
1073   auto TestRequires = [&](int Line) {
1074     const auto BaseTokens = annotate(BaseCode);
1075     const auto ConstrainedTokens = annotate(ConstrainedCode);
1076 
1077 #define LINE " (Line " << Line << ')'
1078 
1079     ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;
1080     ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)
1081         << LINE;
1082 
1083     for (auto I = 0u; I < BaseTokenCount; ++I) {
1084       EXPECT_EQ(
1085           *BaseTokens[I],
1086           *ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])
1087           << I << LINE;
1088     }
1089 
1090 #undef LINE
1091   };
1092 
1093   BaseCode = "template<typename T>\n"
1094              "T Pi = 3.14;";
1095   ConstrainedCode = "template<typename T>\n"
1096                     "  requires Foo<T>\n"
1097                     "T Pi = 3.14;";
1098   BaseTokenCount = 11;
1099   RequiresTokenCount = 5;
1100   PrefixTokenCount = 5;
1101   TestRequires(__LINE__);
1102 
1103   BaseCode = "template<typename T>\n"
1104              "struct Bar;";
1105   ConstrainedCode = "template<typename T>\n"
1106                     "  requires Foo<T>\n"
1107                     "struct Bar;";
1108   BaseTokenCount = 9;
1109   TestRequires(__LINE__);
1110 
1111   BaseCode = "template<typename T>\n"
1112              "struct Bar {\n"
1113              "  T foo();\n"
1114              "  T bar();\n"
1115              "};";
1116   ConstrainedCode = "template<typename T>\n"
1117                     "  requires Foo<T>\n"
1118                     "struct Bar {\n"
1119                     "  T foo();\n"
1120                     "  T bar();\n"
1121                     "};";
1122   BaseTokenCount = 21;
1123   TestRequires(__LINE__);
1124 
1125   BaseCode = "template<typename T>\n"
1126              "Bar(T) -> Bar<T>;";
1127   ConstrainedCode = "template<typename T>\n"
1128                     "  requires Foo<T>\n"
1129                     "Bar(T) -> Bar<T>;";
1130   BaseTokenCount = 16;
1131   TestRequires(__LINE__);
1132 
1133   BaseCode = "template<typename T>\n"
1134              "T foo();";
1135   ConstrainedCode = "template<typename T>\n"
1136                     "  requires Foo<T>\n"
1137                     "T foo();";
1138   BaseTokenCount = 11;
1139   TestRequires(__LINE__);
1140 
1141   BaseCode = "template<typename T>\n"
1142              "T foo() {\n"
1143              "  auto bar = baz();\n"
1144              "  return bar + T{};\n"
1145              "}";
1146   ConstrainedCode = "template<typename T>\n"
1147                     "  requires Foo<T>\n"
1148                     "T foo() {\n"
1149                     "  auto bar = baz();\n"
1150                     "  return bar + T{};\n"
1151                     "}";
1152   BaseTokenCount = 26;
1153   TestRequires(__LINE__);
1154 
1155   BaseCode = "template<typename T>\n"
1156              "T foo();";
1157   ConstrainedCode = "template<typename T>\n"
1158                     "T foo() requires Foo<T>;";
1159   BaseTokenCount = 11;
1160   PrefixTokenCount = 9;
1161   TestRequires(__LINE__);
1162 
1163   BaseCode = "template<typename T>\n"
1164              "T foo() {\n"
1165              "  auto bar = baz();\n"
1166              "  return bar + T{};\n"
1167              "}";
1168   ConstrainedCode = "template<typename T>\n"
1169                     "T foo() requires Foo<T> {\n"
1170                     "  auto bar = baz();\n"
1171                     "  return bar + T{};\n"
1172                     "}";
1173   BaseTokenCount = 26;
1174   TestRequires(__LINE__);
1175 
1176   BaseCode = "template<typename T>\n"
1177              "T foo();";
1178   ConstrainedCode = "template<typename T>\n"
1179                     "  requires(Foo<T>)\n"
1180                     "T foo();";
1181   BaseTokenCount = 11;
1182   RequiresTokenCount = 7;
1183   PrefixTokenCount = 5;
1184   TestRequires(__LINE__);
1185 
1186   BaseCode = "template<typename T>\n"
1187              "Bar(T) -> Bar<typename T::I>;";
1188   ConstrainedCode = "template<typename T>\n"
1189                     "  requires requires(T &&t) {\n"
1190                     "             typename T::I;\n"
1191                     "           }\n"
1192                     "Bar(T) -> Bar<typename T::I>;";
1193   BaseTokenCount = 19;
1194   RequiresTokenCount = 14;
1195   PrefixTokenCount = 5;
1196   TestRequires(__LINE__);
1197 
1198   BaseCode = "struct [[nodiscard]] zero_t {\n"
1199              "  template<class T>\n"
1200              "  [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1201              "};";
1202   ConstrainedCode =
1203       "struct [[nodiscard]] zero_t {\n"
1204       "  template<class T>\n"
1205       "    requires requires { v<T>; }\n"
1206       "  [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1207       "};";
1208   BaseTokenCount = 35;
1209   RequiresTokenCount = 9;
1210   PrefixTokenCount = 13;
1211   TestRequires(__LINE__);
1212 
1213   BaseCode = "constexpr Foo(Foo const &other)\n"
1214              "    : value{other.value} {\n"
1215              "  do_magic();\n"
1216              "  do_more_magic();\n"
1217              "}";
1218   ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1219                     "  requires std::is_copy_constructible<T>\n"
1220                     "    : value{other.value} {\n"
1221                     "  do_magic();\n"
1222                     "  do_more_magic();\n"
1223                     "}";
1224   BaseTokenCount = 26;
1225   RequiresTokenCount = 7;
1226   PrefixTokenCount = 8;
1227   TestRequires(__LINE__);
1228 
1229   BaseCode = "constexpr Foo(Foo const &other)\n"
1230              "    : value{other.value} {\n"
1231              "  do_magic();\n"
1232              "  do_more_magic();\n"
1233              "}";
1234   ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1235                     "  requires (std::is_copy_constructible<T>)\n"
1236                     "    : value{other.value} {\n"
1237                     "  do_magic();\n"
1238                     "  do_more_magic();\n"
1239                     "}";
1240   RequiresTokenCount = 9;
1241   TestRequires(__LINE__);
1242 }
1243 
1244 TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
1245   auto Tokens = annotate("__asm{\n"
1246                          "a:\n"
1247                          "};");
1248   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1249   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1250   EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
1251   EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace);
1252 }
1253 
1254 TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
1255   auto Tokens = annotate("int (^)() = ^ ()\n"
1256                          "  external_source_symbol() { //\n"
1257                          "  return 1;\n"
1258                          "};");
1259   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1260   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1261   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
1262 
1263   Tokens = annotate("int *p = ^int*(){ //\n"
1264                     "  return nullptr;\n"
1265                     "}();");
1266   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1267   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
1268 }
1269 
1270 TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodExpr) {
1271   auto Tokens = annotate("void f() {\n"
1272                          "  //\n"
1273                          "  BOOL a = [b.c n] > 1;\n"
1274                          "}");
1275   EXPECT_EQ(Tokens.size(), 20u) << Tokens;
1276   EXPECT_TOKEN(Tokens[9], tok::l_square, TT_ObjCMethodExpr);
1277   EXPECT_TOKEN(Tokens[15], tok::greater, TT_BinaryOperator);
1278 }
1279 
1280 TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
1281   auto Tokens = annotate("[]() constexpr {}");
1282   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1283   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1284   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1285 
1286   Tokens = annotate("[]() consteval {}");
1287   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1288   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1289   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1290 
1291   Tokens = annotate("[]() mutable {}");
1292   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1293   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1294   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1295 
1296   Tokens = annotate("[]() static {}");
1297   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1298   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1299   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1300 
1301   Tokens = annotate("[]() -> auto {}");
1302   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1303   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1304   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1305   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1306 
1307   Tokens = annotate("[]() -> auto & {}");
1308   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1309   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1310   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1311   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1312 
1313   Tokens = annotate("[]() -> auto * {}");
1314   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1315   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1316   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1317   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1318 
1319   Tokens = annotate("[] {}");
1320   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1321   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1322   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
1323 
1324   Tokens = annotate("[] noexcept {}");
1325   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1326   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1327   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
1328 
1329   Tokens = annotate("[] -> auto {}");
1330   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1331   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1332   EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
1333   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
1334 
1335   Tokens = annotate("[] <typename T> () {}");
1336   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1337   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1338   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1339   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
1340 
1341   Tokens = annotate("[] <typename T> {}");
1342   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1343   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1344   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1345   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1346 
1347   Tokens = annotate("[] <typename... T> () {}");
1348   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1349   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1350   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1351   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1352 
1353   Tokens = annotate("[] <typename... T> {}");
1354   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1355   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1356   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1357   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1358 
1359   Tokens = annotate("[] <int... T> () {}");
1360   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1361   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1362   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1363   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1364 
1365   Tokens = annotate("[] <int... T> {}");
1366   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1367   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1368   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1369   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1370 
1371   Tokens = annotate("[] <Foo... T> () {}");
1372   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1373   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1374   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1375   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1376 
1377   Tokens = annotate("[] <Foo... T> {}");
1378   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1379   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1380   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1381   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1382 
1383   // Lambdas with a requires-clause
1384   Tokens = annotate("[] <typename T> (T t) requires Bar<T> {}");
1385   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1386   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1387   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1388   EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
1389   EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
1390   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1391 
1392   Tokens = annotate("[] <typename T> (T &&t) requires Bar<T> {}");
1393   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1394   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1395   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1396   EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);
1397   EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
1398   EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
1399   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
1400 
1401   Tokens = annotate("[] <typename T> (T t) requires Foo<T> || Bar<T> {}");
1402   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1403   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1404   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1405   EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
1406   EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
1407   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1408 
1409   Tokens = annotate("[] <typename T> (T t) -> T requires Bar<T> {}");
1410   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1411   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1412   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1413   EXPECT_TOKEN(Tokens[10], tok::arrow, TT_LambdaArrow);
1414   EXPECT_TOKEN(Tokens[12], tok::kw_requires, TT_RequiresClause);
1415   EXPECT_TRUE(Tokens[16]->ClosesRequiresClause);
1416   EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
1417 
1418   Tokens = annotate("[] <typename T> requires Bar<T> (T t) {}");
1419   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1420   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1421   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1422   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1423   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1424   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1425 
1426   Tokens = annotate("[] <typename T> requires Bar<T> (T &&t) {}");
1427   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1428   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1429   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1430   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1431   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1432   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_PointerOrReference);
1433   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
1434 
1435   Tokens = annotate("[] <typename T> requires Foo<T> || Bar<T> (T t) {}");
1436   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1437   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1438   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1439   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1440   EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
1441   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1442 
1443   Tokens = annotate("[] <typename T> requires true (T&& t) {}");
1444   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1445   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1446   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1447   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1448   EXPECT_TRUE(Tokens[7]->ClosesRequiresClause);
1449   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1450   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
1451 
1452   Tokens = annotate("[] <typename T> requires Bar<T> {}");
1453   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1454   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1455   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1456   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1457   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1458   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_LambdaLBrace);
1459 
1460   Tokens = annotate("[] <typename T> requires Bar<T> noexcept {}");
1461   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1462   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1463   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1464   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1465   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1466   EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
1467 
1468   Tokens = annotate("[] <typename T> requires Bar<T> -> T {}");
1469   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1470   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1471   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1472   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1473   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1474   EXPECT_TOKEN(Tokens[11], tok::arrow, TT_LambdaArrow);
1475   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
1476 
1477   Tokens = annotate("[] <typename T> requires Foo<T> (T t) requires Bar<T> {}");
1478   ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1479   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1480   EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1481   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1482   EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1483   EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);
1484   EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
1485   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1486 }
1487 
1488 TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
1489   auto Tokens = annotate("template <typename T>\n"
1490                          "DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
1491                          "string OldFunction(const string &parameter) {}");
1492   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1493   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);
1494 
1495   Tokens = annotate("template <typename T>\n"
1496                     "A(T) noexcept;");
1497   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1498   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
1499 }
1500 
1501 TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
1502   auto Tokens = annotate("void f [[noreturn]] ();");
1503   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1504   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1505 
1506   Tokens = annotate("void f [[noreturn]] () {}");
1507   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1508   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1509 }
1510 
1511 TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {
1512   auto Tokens = annotate("_Generic(x, int: 1, default: 0)");
1513   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1514   EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);
1515   EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);
1516   EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);
1517 }
1518 
1519 TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
1520   auto Tokens = annotate("auto f() -> int;");
1521   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1522   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1523 
1524   Tokens = annotate("auto operator->() -> int;");
1525   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1526   EXPECT_TOKEN(Tokens[2], tok::arrow, TT_OverloadedOperator);
1527   EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
1528 
1529   Tokens = annotate("auto operator++(int) -> int;");
1530   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1531   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
1532 
1533   Tokens = annotate("auto operator=() -> int;");
1534   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1535   EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
1536 
1537   Tokens = annotate("auto operator=(int) -> int;");
1538   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1539   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
1540 
1541   Tokens = annotate("auto foo() -> auto { return Val; }");
1542   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1543   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1544 
1545   Tokens = annotate("struct S { auto bar() const -> int; };");
1546   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1547   EXPECT_TOKEN(Tokens[8], tok::arrow, TT_TrailingReturnArrow);
1548 
1549   // Not trailing return arrows
1550   Tokens = annotate("auto a = b->c;");
1551   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1552   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
1553 
1554   Tokens = annotate("auto a = (b)->c;");
1555   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1556   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
1557 
1558   Tokens = annotate("auto a = b()->c;");
1559   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1560   EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
1561 
1562   Tokens = annotate("auto a = b->c();");
1563   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1564   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
1565 
1566   Tokens = annotate("decltype(auto) a = b()->c;");
1567   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1568   EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
1569 
1570   Tokens = annotate("void f() { auto a = b->c(); }");
1571   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1572   EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
1573 
1574   Tokens = annotate("void f() { auto a = b()->c; }");
1575   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1576   EXPECT_TOKEN(Tokens[11], tok::arrow, TT_Unknown);
1577 
1578   // Mixed
1579   Tokens = annotate("auto f() -> int { auto a = b()->c; }");
1580   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1581   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1582   EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
1583 }
1584 
1585 TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
1586   auto Annotate = [this](llvm::StringRef Code) {
1587     return annotate(Code, getLLVMStyle(FormatStyle::LK_Verilog));
1588   };
1589   // Test that unary operators get labeled as such and that operators like '++'
1590   // don't get split.
1591   tok::TokenKind Unary[] = {tok::plus,  tok::minus,    tok::exclaim,
1592                             tok::tilde, tok::amp,      tok::pipe,
1593                             tok::caret, tok::plusplus, tok::minusminus};
1594   for (auto Kind : Unary) {
1595     auto Tokens =
1596         Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");
1597     ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1598     EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);
1599   }
1600   // Operators formed by joining two operators like '^~'. For some of these
1601   // joined operators, we don't have a separate type, so we only test for their
1602   // precedence.
1603   std::pair<prec::Level, std::string> JoinedBinary[] = {
1604       {prec::Comma, "<->"},       {prec::Assignment, "+="},
1605       {prec::Assignment, "-="},   {prec::Assignment, "*="},
1606       {prec::Assignment, "/="},   {prec::Assignment, "%="},
1607       {prec::Assignment, "&="},   {prec::Assignment, "^="},
1608       {prec::Assignment, "<<="},  {prec::Assignment, ">>="},
1609       {prec::Assignment, "<<<="}, {prec::Assignment, ">>>="},
1610       {prec::LogicalOr, "||"},    {prec::LogicalAnd, "&&"},
1611       {prec::Equality, "=="},     {prec::Equality, "!="},
1612       {prec::Equality, "==="},    {prec::Equality, "!=="},
1613       {prec::Equality, "==?"},    {prec::Equality, "!=?"},
1614       {prec::ExclusiveOr, "~^"},  {prec::ExclusiveOr, "^~"},
1615   };
1616   for (auto Operator : JoinedBinary) {
1617     auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");
1618     ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1619     EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);
1620     EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);
1621   }
1622   // '~^' and '^~' can be unary as well as binary operators.
1623   auto Tokens = Annotate("x = ~^x;");
1624   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1625   EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
1626   Tokens = Annotate("x = ^~x;");
1627   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1628   EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
1629   // The unary operators '~&' and '~|' can only be unary operators. The current
1630   // implementation treats each of them as separate unary '~' and '&' or '|'
1631   // operators, which is enough for formatting purposes. In FormatTestVerilog,
1632   // there is a test that there is no space in between. And even if a new line
1633   // is inserted between the '~' and '|', the semantic meaning is the same as
1634   // the joined operator, so the CanBreakBefore property doesn't need to be
1635   // false for the second operator.
1636   Tokens = Annotate("x = ~&x;");
1637   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1638   EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
1639   EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);
1640   Tokens = Annotate("x = ~|x;");
1641   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1642   EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
1643   EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);
1644   // Test for block label colons.
1645   Tokens = Annotate("begin : x\n"
1646                     "end : x");
1647   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1648   EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);
1649   EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);
1650   // Test that the dimension colon is annotated correctly.
1651   Tokens = Annotate("var [1 : 0] x;");
1652   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1653   EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);
1654   Tokens = Annotate("extern function [1 : 0] x;");
1655   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1656   EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
1657   Tokens = Annotate("module test\n"
1658                     "    (input wire [7 : 0] a[7 : 0]);\n"
1659                     "endmodule");
1660   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1661   EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);
1662   EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);
1663   EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);
1664   // Test case labels and ternary operators.
1665   Tokens = Annotate("case (x)\n"
1666                     "  x:\n"
1667                     "    x;\n"
1668                     "endcase\n");
1669   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1670   EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);
1671   Tokens = Annotate("case (x)\n"
1672                     "  x ? x : x:\n"
1673                     "    x;\n"
1674                     "endcase\n");
1675   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1676   EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
1677   EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
1678   EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);
1679   // Non-blocking assignments.
1680   Tokens = Annotate("a <= b;");
1681   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1682   EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
1683   EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
1684   Tokens = Annotate("if (a <= b) break;");
1685   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1686   EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
1687   EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
1688   Tokens = Annotate("a <= b <= a;");
1689   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1690   EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
1691   EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
1692   EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
1693   EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
1694 
1695   // Port lists in module instantiation.
1696   Tokens = Annotate("module_x instance_1(port_1), instance_2(port_2);");
1697   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1698   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
1699   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_VerilogInstancePortLParen);
1700   Tokens = Annotate("module_x #(parameter) instance_1(port_1), "
1701                     "instance_2(port_2);");
1702   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1703   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
1704   EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogInstancePortLParen);
1705   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_VerilogInstancePortLParen);
1706 
1707   // Condition parentheses.
1708   Tokens = Annotate("assert (x);");
1709   ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1710   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
1711   Tokens = Annotate("assert #0 (x);");
1712   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1713   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_ConditionLParen);
1714   Tokens = Annotate("assert final (x);");
1715   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1716   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
1717   Tokens = Annotate("foreach (x[x]) continue;");
1718   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1719   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
1720   Tokens = Annotate("repeat (x[x]) continue;");
1721   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1722   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
1723   Tokens = Annotate("case (x) endcase;");
1724   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1725   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
1726 
1727   // Sensitivity list. The TT_Unknown type is clearly not binding for the
1728   // future, please adapt if those tokens get annotated.  This test is only here
1729   // to prevent the comma from being annotated as TT_VerilogInstancePortComma.
1730   Tokens = Annotate("always @(posedge x, posedge y);");
1731   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1732   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_Unknown);
1733   EXPECT_TOKEN(Tokens[5], tok::comma, TT_Unknown);
1734   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
1735 }
1736 
1737 TEST_F(TokenAnnotatorTest, UnderstandConstructors) {
1738   auto Tokens = annotate("Class::Class() : BaseClass(), Member() {}");
1739 
1740   // The TT_Unknown is clearly not binding for the future, please adapt if those
1741   // tokens get annotated.
1742   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1743   EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
1744   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1745   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);
1746   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
1747   EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
1748   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
1749   EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);
1750   EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);
1751   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
1752 
1753   Tokens = annotate("Class::Class() : BaseClass{}, Member{} {}");
1754   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1755   EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
1756   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1757   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);
1758   EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);
1759   EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
1760   EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
1761   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);
1762   EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);
1763   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
1764 }
1765 
1766 TEST_F(TokenAnnotatorTest, UnderstandsConditionParens) {
1767   auto Tokens = annotate("if (x) {}");
1768   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1769   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
1770   Tokens = annotate("if constexpr (x) {}");
1771   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1772   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
1773   Tokens = annotate("if CONSTEXPR (x) {}");
1774   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1775   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
1776   Tokens = annotate("if (x) {} else if (x) {}");
1777   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1778   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
1779   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_ConditionLParen);
1780 }
1781 
1782 TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {
1783   FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
1784 
1785   auto Tokens = annotate("int? a;", Style);
1786   EXPECT_EQ(Tokens.size(), 5u) << Tokens;
1787   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
1788 
1789   Tokens = annotate("int? a = 1;", Style);
1790   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
1791   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
1792 
1793   Tokens = annotate("int?)", Style);
1794   EXPECT_EQ(Tokens.size(), 4u) << Tokens;
1795   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
1796 
1797   Tokens = annotate("int?>", Style);
1798   EXPECT_EQ(Tokens.size(), 4u) << Tokens;
1799   EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
1800 
1801   Tokens = annotate("cond? id : id2", Style);
1802   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
1803   EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
1804 
1805   Tokens = annotate("cond ? cond2 ? : id1 : id2", Style);
1806   EXPECT_EQ(Tokens.size(), 9u) << Tokens;
1807   EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
1808 }
1809 
1810 TEST_F(TokenAnnotatorTest, UnderstandsLabels) {
1811   auto Tokens = annotate("{ x: break; }");
1812   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1813   EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
1814   Tokens = annotate("{ case x: break; }");
1815   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1816   EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
1817   Tokens = annotate("{ x: { break; } }");
1818   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1819   EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
1820   Tokens = annotate("{ case x: { break; } }");
1821   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1822   EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
1823 }
1824 
1825 TEST_F(TokenAnnotatorTest, UnderstandsNestedBlocks) {
1826   // The closing braces are not annotated. It doesn't seem to cause a problem.
1827   // So we only test for the opening braces.
1828   auto Tokens = annotate("{\n"
1829                          "  {\n"
1830                          "    { int a = 0; }\n"
1831                          "  }\n"
1832                          "  {}\n"
1833                          "}");
1834   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1835   EXPECT_BRACE_KIND(Tokens[0], BK_Block);
1836   EXPECT_BRACE_KIND(Tokens[1], BK_Block);
1837   EXPECT_BRACE_KIND(Tokens[2], BK_Block);
1838   EXPECT_BRACE_KIND(Tokens[10], BK_Block);
1839 }
1840 
1841 } // namespace
1842 } // namespace format
1843 } // namespace clang
1844