xref: /llvm-project/clang/lib/ASTMatchers/Dynamic/Parser.cpp (revision 06becf8f99ca7c8b35a36ba860312ea4537d2c47) 
1 //===- Parser.cpp - Matcher expression parser -----------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Recursive parser implementation for the matcher expression grammar.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/ASTMatchers/Dynamic/Parser.h"
16 #include "clang/ASTMatchers/ASTMatchersInternal.h"
17 #include "clang/ASTMatchers/Dynamic/Diagnostics.h"
18 #include "clang/ASTMatchers/Dynamic/Registry.h"
19 #include "clang/Basic/CharInfo.h"
20 #include "llvm/ADT/Optional.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/ManagedStatic.h"
24 #include <algorithm>
25 #include <cassert>
26 #include <cerrno>
27 #include <cstddef>
28 #include <cstdlib>
29 #include <string>
30 #include <utility>
31 #include <vector>
32 
33 namespace clang {
34 namespace ast_matchers {
35 namespace dynamic {
36 
37 /// \brief Simple structure to hold information for one token from the parser.
38 struct Parser::TokenInfo {
39   /// \brief Different possible tokens.
40   enum TokenKind {
41     TK_Eof,
42     TK_OpenParen,
43     TK_CloseParen,
44     TK_Comma,
45     TK_Period,
46     TK_Literal,
47     TK_Ident,
48     TK_InvalidChar,
49     TK_Error,
50     TK_CodeCompletion
51   };
52 
53   /// \brief Some known identifiers.
54   static const char* const ID_Bind;
55 
56   TokenInfo() = default;
57 
58   StringRef Text;
59   TokenKind Kind = TK_Eof;
60   SourceRange Range;
61   VariantValue Value;
62 };
63 
64 const char* const Parser::TokenInfo::ID_Bind = "bind";
65 
66 /// \brief Simple tokenizer for the parser.
67 class Parser::CodeTokenizer {
68 public:
69   explicit CodeTokenizer(StringRef MatcherCode, Diagnostics *Error)
70       : Code(MatcherCode), StartOfLine(MatcherCode), Error(Error) {
71     NextToken = getNextToken();
72   }
73 
74   CodeTokenizer(StringRef MatcherCode, Diagnostics *Error,
75                 unsigned CodeCompletionOffset)
76       : Code(MatcherCode), StartOfLine(MatcherCode), Error(Error),
77         CodeCompletionLocation(MatcherCode.data() + CodeCompletionOffset) {
78     NextToken = getNextToken();
79   }
80 
81   /// \brief Returns but doesn't consume the next token.
82   const TokenInfo &peekNextToken() const { return NextToken; }
83 
84   /// \brief Consumes and returns the next token.
85   TokenInfo consumeNextToken() {
86     TokenInfo ThisToken = NextToken;
87     NextToken = getNextToken();
88     return ThisToken;
89   }
90 
91   TokenInfo::TokenKind nextTokenKind() const { return NextToken.Kind; }
92 
93 private:
94   TokenInfo getNextToken() {
95     consumeWhitespace();
96     TokenInfo Result;
97     Result.Range.Start = currentLocation();
98 
99     if (CodeCompletionLocation && CodeCompletionLocation <= Code.data()) {
100       Result.Kind = TokenInfo::TK_CodeCompletion;
101       Result.Text = StringRef(CodeCompletionLocation, 0);
102       CodeCompletionLocation = nullptr;
103       return Result;
104     }
105 
106     if (Code.empty()) {
107       Result.Kind = TokenInfo::TK_Eof;
108       Result.Text = "";
109       return Result;
110     }
111 
112     switch (Code[0]) {
113     case ',':
114       Result.Kind = TokenInfo::TK_Comma;
115       Result.Text = Code.substr(0, 1);
116       Code = Code.drop_front();
117       break;
118     case '.':
119       Result.Kind = TokenInfo::TK_Period;
120       Result.Text = Code.substr(0, 1);
121       Code = Code.drop_front();
122       break;
123     case '(':
124       Result.Kind = TokenInfo::TK_OpenParen;
125       Result.Text = Code.substr(0, 1);
126       Code = Code.drop_front();
127       break;
128     case ')':
129       Result.Kind = TokenInfo::TK_CloseParen;
130       Result.Text = Code.substr(0, 1);
131       Code = Code.drop_front();
132       break;
133 
134     case '"':
135     case '\'':
136       // Parse a string literal.
137       consumeStringLiteral(&Result);
138       break;
139 
140     case '0': case '1': case '2': case '3': case '4':
141     case '5': case '6': case '7': case '8': case '9':
142       // Parse an unsigned and float literal.
143       consumeNumberLiteral(&Result);
144       break;
145 
146     default:
147       if (isAlphanumeric(Code[0])) {
148         // Parse an identifier
149         size_t TokenLength = 1;
150         while (true) {
151           // A code completion location in/immediately after an identifier will
152           // cause the portion of the identifier before the code completion
153           // location to become a code completion token.
154           if (CodeCompletionLocation == Code.data() + TokenLength) {
155             CodeCompletionLocation = nullptr;
156             Result.Kind = TokenInfo::TK_CodeCompletion;
157             Result.Text = Code.substr(0, TokenLength);
158             Code = Code.drop_front(TokenLength);
159             return Result;
160           }
161           if (TokenLength == Code.size() || !isAlphanumeric(Code[TokenLength]))
162             break;
163           ++TokenLength;
164         }
165         if (TokenLength == 4 && Code.startswith("true")) {
166           Result.Kind = TokenInfo::TK_Literal;
167           Result.Value = true;
168         } else if (TokenLength == 5 && Code.startswith("false")) {
169           Result.Kind = TokenInfo::TK_Literal;
170           Result.Value = false;
171         } else {
172           Result.Kind = TokenInfo::TK_Ident;
173           Result.Text = Code.substr(0, TokenLength);
174         }
175         Code = Code.drop_front(TokenLength);
176       } else {
177         Result.Kind = TokenInfo::TK_InvalidChar;
178         Result.Text = Code.substr(0, 1);
179         Code = Code.drop_front(1);
180       }
181       break;
182     }
183 
184     Result.Range.End = currentLocation();
185     return Result;
186   }
187 
188   /// \brief Consume an unsigned and float literal.
189   void consumeNumberLiteral(TokenInfo *Result) {
190     bool isFloatingLiteral = false;
191     unsigned Length = 1;
192     if (Code.size() > 1) {
193       // Consume the 'x' or 'b' radix modifier, if present.
194       switch (toLowercase(Code[1])) {
195       case 'x': case 'b': Length = 2;
196       }
197     }
198     while (Length < Code.size() && isHexDigit(Code[Length]))
199       ++Length;
200 
201     // Try to recognize a floating point literal.
202     while (Length < Code.size()) {
203       char c = Code[Length];
204       if (c == '-' || c == '+' || c == '.' || isHexDigit(c)) {
205         isFloatingLiteral = true;
206         Length++;
207       } else {
208         break;
209       }
210     }
211 
212     Result->Text = Code.substr(0, Length);
213     Code = Code.drop_front(Length);
214 
215     if (isFloatingLiteral) {
216       char *end;
217       errno = 0;
218       std::string Text = Result->Text.str();
219       double doubleValue = strtod(Text.c_str(), &end);
220       if (*end == 0 && errno == 0) {
221         Result->Kind = TokenInfo::TK_Literal;
222         Result->Value = doubleValue;
223         return;
224       }
225     } else {
226       unsigned Value;
227       if (!Result->Text.getAsInteger(0, Value)) {
228         Result->Kind = TokenInfo::TK_Literal;
229         Result->Value = Value;
230         return;
231       }
232     }
233 
234     SourceRange Range;
235     Range.Start = Result->Range.Start;
236     Range.End = currentLocation();
237     Error->addError(Range, Error->ET_ParserNumberError) << Result->Text;
238     Result->Kind = TokenInfo::TK_Error;
239   }
240 
241   /// \brief Consume a string literal.
242   ///
243   /// \c Code must be positioned at the start of the literal (the opening
244   /// quote). Consumed until it finds the same closing quote character.
245   void consumeStringLiteral(TokenInfo *Result) {
246     bool InEscape = false;
247     const char Marker = Code[0];
248     for (size_t Length = 1, Size = Code.size(); Length != Size; ++Length) {
249       if (InEscape) {
250         InEscape = false;
251         continue;
252       }
253       if (Code[Length] == '\\') {
254         InEscape = true;
255         continue;
256       }
257       if (Code[Length] == Marker) {
258         Result->Kind = TokenInfo::TK_Literal;
259         Result->Text = Code.substr(0, Length + 1);
260         Result->Value = Code.substr(1, Length - 1);
261         Code = Code.drop_front(Length + 1);
262         return;
263       }
264     }
265 
266     StringRef ErrorText = Code;
267     Code = Code.drop_front(Code.size());
268     SourceRange Range;
269     Range.Start = Result->Range.Start;
270     Range.End = currentLocation();
271     Error->addError(Range, Error->ET_ParserStringError) << ErrorText;
272     Result->Kind = TokenInfo::TK_Error;
273   }
274 
275   /// \brief Consume all leading whitespace from \c Code.
276   void consumeWhitespace() {
277     while (!Code.empty() && isWhitespace(Code[0])) {
278       if (Code[0] == '\n') {
279         ++Line;
280         StartOfLine = Code.drop_front();
281       }
282       Code = Code.drop_front();
283     }
284   }
285 
286   SourceLocation currentLocation() {
287     SourceLocation Location;
288     Location.Line = Line;
289     Location.Column = Code.data() - StartOfLine.data() + 1;
290     return Location;
291   }
292 
293   StringRef Code;
294   StringRef StartOfLine;
295   unsigned Line = 1;
296   Diagnostics *Error;
297   TokenInfo NextToken;
298   const char *CodeCompletionLocation = nullptr;
299 };
300 
301 Parser::Sema::~Sema() = default;
302 
303 std::vector<ArgKind> Parser::Sema::getAcceptedCompletionTypes(
304     llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> Context) {
305   return {};
306 }
307 
308 std::vector<MatcherCompletion>
309 Parser::Sema::getMatcherCompletions(llvm::ArrayRef<ArgKind> AcceptedTypes) {
310   return {};
311 }
312 
313 struct Parser::ScopedContextEntry {
314   Parser *P;
315 
316   ScopedContextEntry(Parser *P, MatcherCtor C) : P(P) {
317     P->ContextStack.push_back(std::make_pair(C, 0u));
318   }
319 
320   ~ScopedContextEntry() {
321     P->ContextStack.pop_back();
322   }
323 
324   void nextArg() {
325     ++P->ContextStack.back().second;
326   }
327 };
328 
329 /// \brief Parse expressions that start with an identifier.
330 ///
331 /// This function can parse named values and matchers.
332 /// In case of failure it will try to determine the user's intent to give
333 /// an appropriate error message.
334 bool Parser::parseIdentifierPrefixImpl(VariantValue *Value) {
335   const TokenInfo NameToken = Tokenizer->consumeNextToken();
336 
337   if (Tokenizer->nextTokenKind() != TokenInfo::TK_OpenParen) {
338     // Parse as a named value.
339     if (const VariantValue NamedValue =
340             NamedValues ? NamedValues->lookup(NameToken.Text)
341                         : VariantValue()) {
342       *Value = NamedValue;
343       return true;
344     }
345     // If the syntax is correct and the name is not a matcher either, report
346     // unknown named value.
347     if ((Tokenizer->nextTokenKind() == TokenInfo::TK_Comma ||
348          Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen ||
349          Tokenizer->nextTokenKind() == TokenInfo::TK_Eof) &&
350         !S->lookupMatcherCtor(NameToken.Text)) {
351       Error->addError(NameToken.Range, Error->ET_RegistryValueNotFound)
352           << NameToken.Text;
353       return false;
354     }
355     // Otherwise, fallback to the matcher parser.
356   }
357 
358   // Parse as a matcher expression.
359   return parseMatcherExpressionImpl(NameToken, Value);
360 }
361 
362 /// \brief Parse and validate a matcher expression.
363 /// \return \c true on success, in which case \c Value has the matcher parsed.
364 ///   If the input is malformed, or some argument has an error, it
365 ///   returns \c false.
366 bool Parser::parseMatcherExpressionImpl(const TokenInfo &NameToken,
367                                         VariantValue *Value) {
368   assert(NameToken.Kind == TokenInfo::TK_Ident);
369   const TokenInfo OpenToken = Tokenizer->consumeNextToken();
370   if (OpenToken.Kind != TokenInfo::TK_OpenParen) {
371     Error->addError(OpenToken.Range, Error->ET_ParserNoOpenParen)
372         << OpenToken.Text;
373     return false;
374   }
375 
376   llvm::Optional<MatcherCtor> Ctor = S->lookupMatcherCtor(NameToken.Text);
377 
378   if (!Ctor) {
379     Error->addError(NameToken.Range, Error->ET_RegistryMatcherNotFound)
380         << NameToken.Text;
381     // Do not return here. We need to continue to give completion suggestions.
382   }
383 
384   std::vector<ParserValue> Args;
385   TokenInfo EndToken;
386 
387   {
388     ScopedContextEntry SCE(this, Ctor ? *Ctor : nullptr);
389 
390     while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
391       if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
392         // End of args.
393         EndToken = Tokenizer->consumeNextToken();
394         break;
395       }
396       if (!Args.empty()) {
397         // We must find a , token to continue.
398         const TokenInfo CommaToken = Tokenizer->consumeNextToken();
399         if (CommaToken.Kind != TokenInfo::TK_Comma) {
400           Error->addError(CommaToken.Range, Error->ET_ParserNoComma)
401               << CommaToken.Text;
402           return false;
403         }
404       }
405 
406       Diagnostics::Context Ctx(Diagnostics::Context::MatcherArg, Error,
407                                NameToken.Text, NameToken.Range,
408                                Args.size() + 1);
409       ParserValue ArgValue;
410       ArgValue.Text = Tokenizer->peekNextToken().Text;
411       ArgValue.Range = Tokenizer->peekNextToken().Range;
412       if (!parseExpressionImpl(&ArgValue.Value)) {
413         return false;
414       }
415 
416       Args.push_back(ArgValue);
417       SCE.nextArg();
418     }
419   }
420 
421   if (EndToken.Kind == TokenInfo::TK_Eof) {
422     Error->addError(OpenToken.Range, Error->ET_ParserNoCloseParen);
423     return false;
424   }
425 
426   std::string BindID;
427   if (Tokenizer->peekNextToken().Kind == TokenInfo::TK_Period) {
428     // Parse .bind("foo")
429     Tokenizer->consumeNextToken();  // consume the period.
430     const TokenInfo BindToken = Tokenizer->consumeNextToken();
431     if (BindToken.Kind == TokenInfo::TK_CodeCompletion) {
432       addCompletion(BindToken, MatcherCompletion("bind(\"", "bind", 1));
433       return false;
434     }
435 
436     const TokenInfo OpenToken = Tokenizer->consumeNextToken();
437     const TokenInfo IDToken = Tokenizer->consumeNextToken();
438     const TokenInfo CloseToken = Tokenizer->consumeNextToken();
439 
440     // TODO: We could use different error codes for each/some to be more
441     //       explicit about the syntax error.
442     if (BindToken.Kind != TokenInfo::TK_Ident ||
443         BindToken.Text != TokenInfo::ID_Bind) {
444       Error->addError(BindToken.Range, Error->ET_ParserMalformedBindExpr);
445       return false;
446     }
447     if (OpenToken.Kind != TokenInfo::TK_OpenParen) {
448       Error->addError(OpenToken.Range, Error->ET_ParserMalformedBindExpr);
449       return false;
450     }
451     if (IDToken.Kind != TokenInfo::TK_Literal || !IDToken.Value.isString()) {
452       Error->addError(IDToken.Range, Error->ET_ParserMalformedBindExpr);
453       return false;
454     }
455     if (CloseToken.Kind != TokenInfo::TK_CloseParen) {
456       Error->addError(CloseToken.Range, Error->ET_ParserMalformedBindExpr);
457       return false;
458     }
459     BindID = IDToken.Value.getString();
460   }
461 
462   if (!Ctor)
463     return false;
464 
465   // Merge the start and end infos.
466   Diagnostics::Context Ctx(Diagnostics::Context::ConstructMatcher, Error,
467                            NameToken.Text, NameToken.Range);
468   SourceRange MatcherRange = NameToken.Range;
469   MatcherRange.End = EndToken.Range.End;
470   VariantMatcher Result = S->actOnMatcherExpression(
471       *Ctor, MatcherRange, BindID, Args, Error);
472   if (Result.isNull()) return false;
473 
474   *Value = Result;
475   return true;
476 }
477 
478 // If the prefix of this completion matches the completion token, add it to
479 // Completions minus the prefix.
480 void Parser::addCompletion(const TokenInfo &CompToken,
481                            const MatcherCompletion& Completion) {
482   if (StringRef(Completion.TypedText).startswith(CompToken.Text) &&
483       Completion.Specificity > 0) {
484     Completions.emplace_back(Completion.TypedText.substr(CompToken.Text.size()),
485                              Completion.MatcherDecl, Completion.Specificity);
486   }
487 }
488 
489 std::vector<MatcherCompletion> Parser::getNamedValueCompletions(
490     ArrayRef<ArgKind> AcceptedTypes) {
491   if (!NamedValues) return std::vector<MatcherCompletion>();
492   std::vector<MatcherCompletion> Result;
493   for (const auto &Entry : *NamedValues) {
494     unsigned Specificity;
495     if (Entry.getValue().isConvertibleTo(AcceptedTypes, &Specificity)) {
496       std::string Decl =
497           (Entry.getValue().getTypeAsString() + " " + Entry.getKey()).str();
498       Result.emplace_back(Entry.getKey(), Decl, Specificity);
499     }
500   }
501   return Result;
502 }
503 
504 void Parser::addExpressionCompletions() {
505   const TokenInfo CompToken = Tokenizer->consumeNextToken();
506   assert(CompToken.Kind == TokenInfo::TK_CodeCompletion);
507 
508   // We cannot complete code if there is an invalid element on the context
509   // stack.
510   for (ContextStackTy::iterator I = ContextStack.begin(),
511                                 E = ContextStack.end();
512        I != E; ++I) {
513     if (!I->first)
514       return;
515   }
516 
517   auto AcceptedTypes = S->getAcceptedCompletionTypes(ContextStack);
518   for (const auto &Completion : S->getMatcherCompletions(AcceptedTypes)) {
519     addCompletion(CompToken, Completion);
520   }
521 
522   for (const auto &Completion : getNamedValueCompletions(AcceptedTypes)) {
523     addCompletion(CompToken, Completion);
524   }
525 }
526 
527 /// \brief Parse an <Expresssion>
528 bool Parser::parseExpressionImpl(VariantValue *Value) {
529   switch (Tokenizer->nextTokenKind()) {
530   case TokenInfo::TK_Literal:
531     *Value = Tokenizer->consumeNextToken().Value;
532     return true;
533 
534   case TokenInfo::TK_Ident:
535     return parseIdentifierPrefixImpl(Value);
536 
537   case TokenInfo::TK_CodeCompletion:
538     addExpressionCompletions();
539     return false;
540 
541   case TokenInfo::TK_Eof:
542     Error->addError(Tokenizer->consumeNextToken().Range,
543                     Error->ET_ParserNoCode);
544     return false;
545 
546   case TokenInfo::TK_Error:
547     // This error was already reported by the tokenizer.
548     return false;
549 
550   case TokenInfo::TK_OpenParen:
551   case TokenInfo::TK_CloseParen:
552   case TokenInfo::TK_Comma:
553   case TokenInfo::TK_Period:
554   case TokenInfo::TK_InvalidChar:
555     const TokenInfo Token = Tokenizer->consumeNextToken();
556     Error->addError(Token.Range, Error->ET_ParserInvalidToken) << Token.Text;
557     return false;
558   }
559 
560   llvm_unreachable("Unknown token kind.");
561 }
562 
563 static llvm::ManagedStatic<Parser::RegistrySema> DefaultRegistrySema;
564 
565 Parser::Parser(CodeTokenizer *Tokenizer, Sema *S,
566                const NamedValueMap *NamedValues, Diagnostics *Error)
567     : Tokenizer(Tokenizer), S(S ? S : &*DefaultRegistrySema),
568       NamedValues(NamedValues), Error(Error) {}
569 
570 Parser::RegistrySema::~RegistrySema() = default;
571 
572 llvm::Optional<MatcherCtor>
573 Parser::RegistrySema::lookupMatcherCtor(StringRef MatcherName) {
574   return Registry::lookupMatcherCtor(MatcherName);
575 }
576 
577 VariantMatcher Parser::RegistrySema::actOnMatcherExpression(
578     MatcherCtor Ctor, SourceRange NameRange, StringRef BindID,
579     ArrayRef<ParserValue> Args, Diagnostics *Error) {
580   if (BindID.empty()) {
581     return Registry::constructMatcher(Ctor, NameRange, Args, Error);
582   } else {
583     return Registry::constructBoundMatcher(Ctor, NameRange, BindID, Args,
584                                            Error);
585   }
586 }
587 
588 std::vector<ArgKind> Parser::RegistrySema::getAcceptedCompletionTypes(
589     ArrayRef<std::pair<MatcherCtor, unsigned>> Context) {
590   return Registry::getAcceptedCompletionTypes(Context);
591 }
592 
593 std::vector<MatcherCompletion> Parser::RegistrySema::getMatcherCompletions(
594     ArrayRef<ArgKind> AcceptedTypes) {
595   return Registry::getMatcherCompletions(AcceptedTypes);
596 }
597 
598 bool Parser::parseExpression(StringRef Code, Sema *S,
599                              const NamedValueMap *NamedValues,
600                              VariantValue *Value, Diagnostics *Error) {
601   CodeTokenizer Tokenizer(Code, Error);
602   if (!Parser(&Tokenizer, S, NamedValues, Error).parseExpressionImpl(Value))
603     return false;
604   if (Tokenizer.peekNextToken().Kind != TokenInfo::TK_Eof) {
605     Error->addError(Tokenizer.peekNextToken().Range,
606                     Error->ET_ParserTrailingCode);
607     return false;
608   }
609   return true;
610 }
611 
612 std::vector<MatcherCompletion>
613 Parser::completeExpression(StringRef Code, unsigned CompletionOffset, Sema *S,
614                            const NamedValueMap *NamedValues) {
615   Diagnostics Error;
616   CodeTokenizer Tokenizer(Code, &Error, CompletionOffset);
617   Parser P(&Tokenizer, S, NamedValues, &Error);
618   VariantValue Dummy;
619   P.parseExpressionImpl(&Dummy);
620 
621   // Sort by specificity, then by name.
622   std::sort(P.Completions.begin(), P.Completions.end(),
623             [](const MatcherCompletion &A, const MatcherCompletion &B) {
624     if (A.Specificity != B.Specificity)
625       return A.Specificity > B.Specificity;
626     return A.TypedText < B.TypedText;
627   });
628 
629   return P.Completions;
630 }
631 
632 llvm::Optional<DynTypedMatcher>
633 Parser::parseMatcherExpression(StringRef Code, Sema *S,
634                                const NamedValueMap *NamedValues,
635                                Diagnostics *Error) {
636   VariantValue Value;
637   if (!parseExpression(Code, S, NamedValues, &Value, Error))
638     return llvm::Optional<DynTypedMatcher>();
639   if (!Value.isMatcher()) {
640     Error->addError(SourceRange(), Error->ET_ParserNotAMatcher);
641     return llvm::Optional<DynTypedMatcher>();
642   }
643   llvm::Optional<DynTypedMatcher> Result =
644       Value.getMatcher().getSingleMatcher();
645   if (!Result.hasValue()) {
646     Error->addError(SourceRange(), Error->ET_ParserOverloadedType)
647         << Value.getTypeAsString();
648   }
649   return Result;
650 }
651 
652 } // namespace dynamic
653 } // namespace ast_matchers
654 } // namespace clang
655