86   lexToken(const char *&First, const char *const End);
88   dependency_directives_scan::Token &lexIncludeFilename(const char *&First,
91   void skipLine(const char *&First, const char *const End);
92   void skipDirective(StringRef Name, const char *&First, const char *const End);
100   tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
103   [[nodiscard]] StringRef lexIdentifier(const char *&First,
112                                                 const char *&First,
115   [[nodiscard]] bool scanImpl(const char *First, const char *const End);
116   [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);
117   [[nodiscard]] bool lexAt(const char *&First, const char *const End);
118   [[nodiscard]] bool lexModule(const char *&First, const char *const End);
119   [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,
121   [[nodiscard]] bool lexPragma(const char *&First, const char *const End);
122   [[nodiscard]] bool lexEndif(const char *&First, const char *const End);
123   [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,
126                                             const char *&First,
128   void lexPPDirectiveBody(const char *&First, const char *const End);
180 static void skipOverSpaces(const char *&First, const char *const End) {  in skipOverSpaces()  argument
181   while (First != End && isHorizontalWhitespace(*First))  in skipOverSpaces()
182     ++First;  in skipOverSpaces()
185 [[nodiscard]] static bool isRawStringLiteral(const char *First,  in isRawStringLiteral()  argument
187   assert(First <= Current);  in isRawStringLiteral()
190   if (*Current != '"' || First == Current)  in isRawStringLiteral()
197   if (First == Current || !isAsciiIdentifierContinue(*--Current))  in isRawStringLiteral()
202     return First == Current || !isAsciiIdentifierContinue(*--Current);  in isRawStringLiteral()
205   if (*Current != '8' || First == Current || *Current-- != 'u')  in isRawStringLiteral()
207   return First == Current || !isAsciiIdentifierContinue(*--Current);  in isRawStringLiteral()
210 static void skipRawString(const char *&First, const char *const End) {  in skipRawString()  argument
211   assert(First[0] == '"');  in skipRawString()
212   assert(First[-1] == 'R');  in skipRawString()
214   const char *Last = ++First;  in skipRawString()
218     First = Last; // Hit the end... just give up.  in skipRawString()
222   StringRef Terminator(First, Last - First);  in skipRawString()
225     First = Last;  in skipRawString()
226     while (First != End && *First != ')')  in skipRawString()
227       ++First;  in skipRawString()
228     if (First == End)  in skipRawString()
230     ++First;  in skipRawString()
233     Last = First;  in skipRawString()
234     while (Last != End && size_t(Last - First) < Terminator.size() &&  in skipRawString()
235            Terminator[Last - First] == *Last)  in skipRawString()
240       First = Last;  in skipRawString()
243     if (size_t(Last - First) < Terminator.size())  in skipRawString()
247     First = Last + 1;  in skipRawString()
253 static unsigned isEOL(const char *First, const char *const End) {  in isEOL()  argument
254   if (First == End)  in isEOL()
256   if (End - First > 1 && isVerticalWhitespace(First[0]) &&  in isEOL()
257       isVerticalWhitespace(First[1]) && First[0] != First[1])  in isEOL()
259   return !!isVerticalWhitespace(First[0]);  in isEOL()
262 static void skipString(const char *&First, const char *const End) {  in skipString()  argument
263   assert(*First == '\'' || *First == '"' || *First == '<');  in skipString()
264   const char Terminator = *First == '<' ? '>' : *First;  in skipString()
265   for (++First; First != End && *First != Terminator; ++First) {  in skipString()
267     if (isVerticalWhitespace(*First))  in skipString()
269     if (*First != '\\')  in skipString()
274     if (++First == End)  in skipString()
276     if (!isWhitespace(*First))  in skipString()
279     const char *FirstAfterBackslashPastSpace = First;  in skipString()
284       First = FirstAfterBackslashPastSpace + NLSize - 1;  in skipString()
287   if (First != End)  in skipString()
288     ++First; // Finish off the string.  in skipString()
292 static unsigned skipNewline(const char *&First, const char *End) {  in skipNewline()  argument
293   if (First == End)  in skipNewline()
295   assert(isVerticalWhitespace(*First));  in skipNewline()
296   unsigned Len = isEOL(First, End);  in skipNewline()
298   First += Len;  in skipNewline()
302 static bool wasLineContinuation(const char *First, unsigned EOLLen) {  in wasLineContinuation()  argument
303   return *(First - (int)EOLLen - 1) == '\\';  in wasLineContinuation()
306 static void skipToNewlineRaw(const char *&First, const char *const End) {  in skipToNewlineRaw()  argument
308     if (First == End)  in skipToNewlineRaw()
311     unsigned Len = isEOL(First, End);  in skipToNewlineRaw()
316       if (++First == End)  in skipToNewlineRaw()
318       Len = isEOL(First, End);  in skipToNewlineRaw()
321     if (First[-1] != '\\')  in skipToNewlineRaw()
324     First += Len;  in skipToNewlineRaw()
329 static void skipLineComment(const char *&First, const char *const End) {  in skipLineComment()  argument
330   assert(First[0] == '/' && First[1] == '/');  in skipLineComment()
331   First += 2;  in skipLineComment()
332   skipToNewlineRaw(First, End);  in skipLineComment()
335 static void skipBlockComment(const char *&First, const char *const End) {  in skipBlockComment()  argument
336   assert(First[0] == '/' && First[1] == '*');  in skipBlockComment()
337   if (End - First < 4) {  in skipBlockComment()
338     First = End;  in skipBlockComment()
341   for (First += 3; First != End; ++First)  in skipBlockComment()
342     if (First[-1] == '*' && First[0] == '/') {  in skipBlockComment()
343       ++First;  in skipBlockComment()
372 void Scanner::skipLine(const char *&First, const char *const End) {  in skipLine()  argument
374     assert(First <= End);  in skipLine()
375     if (First == End)  in skipLine()
378     if (isVerticalWhitespace(*First)) {  in skipLine()
379       skipNewline(First, End);  in skipLine()
382     const char *Start = First;  in skipLine()
383     while (First != End && !isVerticalWhitespace(*First)) {  in skipLine()
385       if (*First == '"' ||  in skipLine()
386           (*First == '\'' && !isQuoteCppDigitSeparator(Start, First, End))) {  in skipLine()
387         LastTokenPtr = First;  in skipLine()
388         if (isRawStringLiteral(Start, First))  in skipLine()
389           skipRawString(First, End);  in skipLine()
391           skipString(First, End);  in skipLine()
396       if (*First != '/' || End - First < 2) {  in skipLine()
397         LastTokenPtr = First;  in skipLine()
398         ++First;  in skipLine()
402       if (First[1] == '/') {  in skipLine()
404         skipLineComment(First, End);  in skipLine()
408       if (First[1] != '*') {  in skipLine()
409         LastTokenPtr = First;  in skipLine()
410         ++First;  in skipLine()
415       skipBlockComment(First, End);  in skipLine()
417     if (First == End)  in skipLine()
421     unsigned Len = skipNewline(First, End);  in skipLine()
422     if (!wasLineContinuation(First, Len)) // Continue past line-continuations.  in skipLine()
427 void Scanner::skipDirective(StringRef Name, const char *&First,  in skipDirective()  argument
434     skipToNewlineRaw(First, End);  in skipDirective()
436     skipLine(First, End);  in skipDirective()
439 static void skipWhitespace(const char *&First, const char *const End) {  in skipWhitespace()  argument
441     assert(First <= End);  in skipWhitespace()
442     skipOverSpaces(First, End);  in skipWhitespace()
444     if (End - First < 2)  in skipWhitespace()
447     if (First[0] == '\\' && isVerticalWhitespace(First[1])) {  in skipWhitespace()
448       skipNewline(++First, End);  in skipWhitespace()
453     if (First[0] != '/')  in skipWhitespace()
457     if (First[1] == '/') {  in skipWhitespace()
458       skipLineComment(First, End);  in skipWhitespace()
463     if (First[1] != '*')  in skipWhitespace()
467     skipBlockComment(First, End);  in skipWhitespace()
471 bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,  in lexModuleDirectiveBody()  argument
475     const dependency_directives_scan::Token &Tok = lexToken(First, End);  in lexModuleDirectiveBody()
484   skipWhitespace(First, End);  in lexModuleDirectiveBody()
485   if (First == End)  in lexModuleDirectiveBody()
487   if (!isVerticalWhitespace(*First))  in lexModuleDirectiveBody()
490   skipNewline(First, End);  in lexModuleDirectiveBody()
494 dependency_directives_scan::Token &Scanner::lexToken(const char *&First,  in lexToken()  argument
498   First = Input.data() + TheLexer.getCurrentBufferOffset();  in lexToken()
499   assert(First <= End);  in lexToken()
508 Scanner::lexIncludeFilename(const char *&First, const char *const End) {  in lexIncludeFilename()  argument
511   First = Input.data() + TheLexer.getCurrentBufferOffset();  in lexIncludeFilename()
512   assert(First <= End);  in lexIncludeFilename()
520 void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {  in lexPPDirectiveBody()  argument
522     const dependency_directives_scan::Token &Tok = lexToken(First, End);  in lexPPDirectiveBody()
529 Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {  in tryLexIdentifierOrSkipLine()  argument
530   const dependency_directives_scan::Token &Tok = lexToken(First, End);  in tryLexIdentifierOrSkipLine()
533       skipLine(First, End);  in tryLexIdentifierOrSkipLine()
558 StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {  in lexIdentifier()  argument
559   std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);  in lexIdentifier()
564 bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,  in isNextIdentifierOrSkipLine()  argument
567           tryLexIdentifierOrSkipLine(First, End)) {  in isNextIdentifierOrSkipLine()
570     skipLine(First, End);  in isNextIdentifierOrSkipLine()
575 bool Scanner::lexAt(const char *&First, const char *const End) {  in lexAt()  argument
579   const dependency_directives_scan::Token &AtTok = lexToken(First, End);  in lexAt()
583   if (!isNextIdentifierOrSkipLine("import", First, End))  in lexAt()
585   return lexModuleDirectiveBody(decl_at_import, First, End);  in lexAt()
588 bool Scanner::lexModule(const char *&First, const char *const End) {  in lexModule()  argument
589   StringRef Id = lexIdentifier(First, End);  in lexModule()
593     std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);  in lexModule()
600     skipLine(First, End);  in lexModule()
604   skipWhitespace(First, End);  in lexModule()
609   switch (*First) {  in lexModule()
615     if (!isAsciiIdentifierContinue(*First)) {  in lexModule()
616       skipLine(First, End);  in lexModule()
621   TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ false);  in lexModule()
629   return lexModuleDirectiveBody(Kind, First, End);  in lexModule()
632 bool Scanner::lexPragma(const char *&First, const char *const End) {  in lexPragma()  argument
633   std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);  in lexPragma()
645     lexPPDirectiveBody(First, End);  in lexPragma()
651     skipLine(First, End);  in lexPragma()
656   if (!isNextIdentifierOrSkipLine("module", First, End))  in lexPragma()
660   if (!isNextIdentifierOrSkipLine("import", First, End))  in lexPragma()
664   lexPPDirectiveBody(First, End);  in lexPragma()
669 bool Scanner::lexEndif(const char *&First, const char *const End) {  in lexEndif()  argument
682     skipLine(First, End);  in lexEndif()
686   return lexDefault(pp_endif, First, End);  in lexEndif()
689 bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,  in lexDefault()  argument
691   lexPPDirectiveBody(First, End);  in lexDefault()
696 static bool isStartOfRelevantLine(char First) {  in isStartOfRelevantLine()  argument
697   switch (First) {  in isStartOfRelevantLine()
708 bool Scanner::lexPPLine(const char *&First, const char *const End) {  in lexPPLine()  argument
709   assert(First != End);  in lexPPLine()
711   skipWhitespace(First, End);  in lexPPLine()
712   assert(First <= End);  in lexPPLine()
713   if (First == End)  in lexPPLine()
716   if (!isStartOfRelevantLine(*First)) {  in lexPPLine()
717     skipLine(First, End);  in lexPPLine()
718     assert(First <= End);  in lexPPLine()
722   LastTokenPtr = First;  in lexPPLine()
724   TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ true);  in lexPPLine()
733   if (*First == '@')  in lexPPLine()
734     return lexAt(First, End);  in lexPPLine()
736   if (*First == 'i' || *First == 'e' || *First == 'm')  in lexPPLine()
737     return lexModule(First, End);  in lexPPLine()
746   const dependency_directives_scan::Token &HashTok = lexToken(First, End);  in lexPPLine()
751     skipLine(First, End);  in lexPPLine()
752     assert(First <= End);  in lexPPLine()
758   std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);  in lexPPLine()
765     return lexPragma(First, End);  in lexPPLine()
784     skipDirective(Id, First, End);  in lexPPLine()
789     return lexEndif(First, End);  in lexPPLine()
796     lexIncludeFilename(First, End);  in lexPPLine()
803   return lexDefault(Kind, First, End);  in lexPPLine()
806 static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {  in skipUTF8ByteOrderMark()  argument
807   if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&  in skipUTF8ByteOrderMark()
808       First[2] == '\xbf')  in skipUTF8ByteOrderMark()
809     First += 3;  in skipUTF8ByteOrderMark()
812 bool Scanner::scanImpl(const char *First, const char *const End) {  in scanImpl()  argument
813   skipUTF8ByteOrderMark(First, End);  in scanImpl()
814   while (First != End)  in scanImpl()
815     if (lexPPLine(First, End))  in scanImpl()