Lines Matching +full:back +full:- +full:end
1 //===--- FormatTokenLexer.cpp - Lex FormatTokens -------------*- C++ ----*-===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
38 Lex->SetKeepWhitespaceMode(true);
100 if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
101 FirstInLineIndex = Tokens.size() - 1;
102 } while (Tokens.back()->isNot(tok::eof));
104 auto &TokEOF = *Tokens.back();
138 Tokens.back()->Tok.setKind(tok::pipepipe);
143 Tokens.back()->Tok.setKind(tok::period);
163 Tokens.back()->Tok.setKind(tok::l_square);
193 Tokens.back()->Tok.setKind(tok::starequal);
199 Tokens.back()->Tok.setKind(tok::equal);
215 if (Tokens.size() >= 3 && Tokens.end()[-3]->is(TT_VerilogNumberBase) &&
216 Tokens.end()[-2]->is(tok::numeric_constant) &&
217 Tokens.back()->isOneOf(tok::numeric_constant, tok::identifier,
231 if (Tokens.back()->TokenText.size() == 1 &&
234 Tokens.back()->Tok.setKind(tok::caret);
239 Tokens.back()->Tok.setKind(tok::lessless);
243 Tokens.back()->Tok.setKind(tok::greatergreater);
253 Tokens.back()->ForcedPrecedence = prec::Assignment;
281 Tokens.back()->is(tok::arrow)) {
282 Tokens.back()->ForcedPrecedence = prec::Comma;
291 Tokens.back()->setFinalizedType(TT_TableGenMultiLineString);
292 Tokens.back()->Tok.setKind(tok::string_literal);
299 Tokens.back()->Tok.setKind(tok::identifier);
300 Tokens.back()->Tok.setIdentifierInfo(nullptr);
301 if (Tokens.back()->TokenText == "!cond")
302 Tokens.back()->setFinalizedType(TT_TableGenCondOperator);
304 Tokens.back()->setFinalizedType(TT_TableGenBangOperator);
310 Tokens.back()->Tok.setKind(tok::identifier);
311 Tokens.back()->Tok.setIdentifierInfo(nullptr);
312 Tokens.back()->setFinalizedType(TT_TableGenBangOperator);
315 // +, - with numbers are literals. Not unary operators.
317 Tokens.back()->Tok.setKind(tok::numeric_constant);
321 Tokens.back()->Tok.setKind(tok::numeric_constant);
330 auto &At = *(Tokens.end() - 2);
331 auto &String = *(Tokens.end() - 1);
332 if (At->isNot(tok::at) || String->isNot(tok::string_literal))
334 At->Tok.setKind(tok::string_literal);
335 At->TokenText = StringRef(At->TokenText.begin(),
336 String->TokenText.end() - At->TokenText.begin());
337 At->ColumnWidth += String->ColumnWidth;
338 At->setType(TT_ObjCStringLiteral);
339 Tokens.erase(Tokens.end() - 1);
348 auto &Hash = *(Tokens.end() - 2);
349 auto &Identifier = *(Tokens.end() - 1);
350 if (Hash->isNot(tok::hash) || Identifier->isNot(tok::identifier))
352 Hash->Tok.setKind(tok::identifier);
353 Hash->TokenText =
354 StringRef(Hash->TokenText.begin(),
355 Identifier->TokenText.end() - Hash->TokenText.begin());
356 Hash->ColumnWidth += Identifier->ColumnWidth;
357 Hash->setType(TT_JsPrivateIdentifier);
358 Tokens.erase(Tokens.end() - 1);
366 // handleCSharpVerbatimAndInterpolatedStrings with lower-level lexing.
372 const auto String = *(Tokens.end() - 1);
373 if (String->isNot(tok::string_literal))
376 auto Prefix = *(Tokens.end() - 2);
377 if (Prefix->isNot(tok::at) && Prefix->TokenText != "$")
381 const auto Tok = *(Tokens.end() - 3);
382 if ((Tok->TokenText == "$" && Prefix->is(tok::at)) ||
383 (Tok->is(tok::at) && Prefix->TokenText == "$")) {
385 Tok->ColumnWidth += Prefix->ColumnWidth;
386 Tokens.erase(Tokens.end() - 2);
391 // Convert back into just a string_literal.
392 Prefix->Tok.setKind(tok::string_literal);
393 Prefix->TokenText =
394 StringRef(Prefix->TokenText.begin(),
395 String->TokenText.end() - Prefix->TokenText.begin());
396 Prefix->ColumnWidth += String->ColumnWidth;
397 Prefix->setType(TT_CSharpStringLiteral);
398 Tokens.erase(Tokens.end() - 1);
403 // https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/attributes/#attribute-targets
412 auto &NullishCoalescing = *(Tokens.end() - 2);
413 auto &Equal = *(Tokens.end() - 1);
414 if (NullishCoalescing->isNot(TT_NullCoalescingOperator) ||
415 Equal->isNot(tok::equal)) {
418 NullishCoalescing->Tok.setKind(tok::equal); // no '??=' in clang tokens.
419 NullishCoalescing->TokenText =
420 StringRef(NullishCoalescing->TokenText.begin(),
421 Equal->TokenText.end() - NullishCoalescing->TokenText.begin());
422 NullishCoalescing->ColumnWidth += Equal->ColumnWidth;
423 NullishCoalescing->setType(TT_NullCoalescingEqual);
424 Tokens.erase(Tokens.end() - 1);
431 const auto At = *(Tokens.end() - 2);
432 if (At->isNot(tok::at))
434 const auto Keyword = *(Tokens.end() - 1);
435 if (Keyword->TokenText == "$")
440 At->Tok.setKind(tok::identifier);
441 At->TokenText = StringRef(At->TokenText.begin(),
442 Keyword->TokenText.end() - At->TokenText.begin());
443 At->ColumnWidth += Keyword->ColumnWidth;
444 At->setType(Keyword->getType());
445 Tokens.erase(Tokens.end() - 1);
453 auto &Identifier = *(Tokens.end() - 1);
454 if (Identifier->isNot(tok::identifier))
456 if (Identifier->TokenText != "foreach")
459 Identifier->setType(TT_ForEachMacro);
460 Identifier->Tok.setKind(tok::kw_for);
467 auto &For = *(Tokens.end() - 2);
468 auto &Each = *(Tokens.end() - 1);
469 if (For->isNot(tok::kw_for))
471 if (Each->isNot(tok::identifier))
473 if (Each->TokenText != "each")
476 For->setType(TT_ForEachMacro);
477 For->Tok.setKind(tok::kw_for);
479 For->TokenText = StringRef(For->TokenText.begin(),
480 Each->TokenText.end() - For->TokenText.begin());
481 For->ColumnWidth += Each->ColumnWidth;
482 Tokens.erase(Tokens.end() - 1);
489 auto &Try = *(Tokens.end() - 2);
490 if (Try->isNot(tok::kw_try))
492 auto &Next = *(Tokens.end() - 1);
493 if (Next->isOneOf(tok::l_brace, tok::colon, tok::hash, tok::comment))
497 auto &At = *(Tokens.end() - 3);
498 if (At->is(tok::at))
502 Try->Tok.setKind(tok::identifier);
511 auto First = Tokens.end() - 3;
512 if (First[0]->isNot(tok::less) || First[1]->isNot(tok::less))
516 if (First[1]->hasWhitespaceBefore())
519 auto X = Tokens.size() > 3 ? First[-1] : nullptr;
520 if (X && X->is(tok::less))
524 if ((!X || X->isNot(tok::kw_operator)) && Y->is(tok::less))
527 First[0]->Tok.setKind(tok::lessless);
528 First[0]->TokenText = "<<";
529 First[0]->ColumnWidth += 1;
530 Tokens.erase(Tokens.end() - 2);
539 auto First = Tokens.end() - 2;
540 if (First[0]->isNot(tok::greater) || First[1]->isNot(tok::greater))
544 if (First[1]->hasWhitespaceBefore())
547 auto Tok = Tokens.size() > 2 ? First[-1] : nullptr;
548 if (Tok && Tok->isNot(tok::kw_operator))
551 First[0]->Tok.setKind(tok::greatergreater);
552 First[0]->TokenText = ">>";
553 First[0]->ColumnWidth += 1;
554 Tokens.erase(Tokens.end() - 1);
564 Tokens.end() - Kinds.size();
566 if (First[i]->isNot(Kinds[i]))
576 SmallVectorImpl<FormatToken *>::const_iterator First = Tokens.end() - Count;
581 if (First[i]->hasWhitespaceBefore())
583 AddLength += First[i]->TokenText.size();
586 Tokens.resize(Tokens.size() - Count + 1);
587 First[0]->TokenText = StringRef(First[0]->TokenText.data(),
588 First[0]->TokenText.size() + AddLength);
589 First[0]->ColumnWidth += AddLength;
590 First[0]->setType(NewType);
606 return Tok->isOneOf(tok::period, tok::l_paren, tok::comma, tok::l_brace,
609 Tok->isOneOf(tok::kw_return, tok::kw_do, tok::kw_case, tok::kw_throw,
612 Tok->isBinaryOperator();
620 // postfix unary operators. If the '++' is followed by a non-operand
623 // `!` is an unary prefix operator, but also a post-fix operator that casts
625 if (Prev->isOneOf(tok::plusplus, tok::minusminus, tok::exclaim))
626 return Tokens.size() < 3 || precedesOperand(Tokens[Tokens.size() - 3]);
641 FormatToken *RegexToken = Tokens.back();
642 if (!RegexToken->isOneOf(tok::slash, tok::slashequal))
649 if (FT->isNot(tok::comment)) {
659 const char *Offset = Lex->getBufferLocation();
660 const char *RegexBegin = Offset - RegexToken->TokenText.size();
661 StringRef Buffer = Lex->getBuffer();
664 for (; !HaveClosingSlash && Offset != Buffer.end(); ++Offset) {
667 // See http://www.ecma-international.org/ecma-262/5.1/#sec-7.8.5.
686 RegexToken->setType(TT_RegexLiteral);
688 RegexToken->Tok.setKind(tok::string_literal);
689 RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
690 RegexToken->ColumnWidth = RegexToken->TokenText.size();
692 resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
695 static auto lexCSharpString(const char *Begin, const char *End, bool Verbatim,
697 auto Repeated = [&Begin, End]() {
698 return Begin + 1 < End && Begin[1] == Begin[0];
707 // single string-literal.
713 for (int UnmatchedOpeningBraceCount = 0; Begin < End; ++Begin) {
734 --UnmatchedOpeningBraceCount;
736 return End;
751 return End;
755 FormatToken *CSharpStringLiteral = Tokens.back();
757 if (CSharpStringLiteral->isNot(TT_CSharpStringLiteral))
760 auto &TokenText = CSharpStringLiteral->TokenText;
777 const char *StrBegin = Lex->getBufferLocation() - TokenText.size();
784 const auto End = Lex->getBuffer().end();
785 Offset = lexCSharpString(Offset, End, Verbatim, Interpolated);
789 if (Offset >= End)
792 StringRef LiteralText(StrBegin, Offset - StrBegin + 1);
800 CSharpStringLiteral->ColumnWidth = encoding::columnWidthWithTabs(
801 FirstLineText, CSharpStringLiteral->OriginalColumn, Style.TabWidth,
805 CSharpStringLiteral->IsMultiline = true;
807 CSharpStringLiteral->LastLineColumnWidth =
812 assert(Offset < End);
813 resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset + 1)));
817 FormatToken *MultiLineString = Tokens.back();
818 if (MultiLineString->isNot(TT_TableGenMultiLineString))
821 auto OpenOffset = Lex->getCurrentBufferOffset() - 2 /* "[{" */;
822 // "}]" is the end of multi line string.
823 auto CloseOffset = Lex->getBuffer().find("}]", OpenOffset);
826 auto Text = Lex->getBuffer().substr(OpenOffset, CloseOffset - OpenOffset + 2);
827 MultiLineString->TokenText = Text;
829 Lex->getSourceLocation(Lex->getBufferLocation() - 2 + Text.size())));
834 MultiLineString->IsMultiline = true;
838 MultiLineString->LastLineColumnWidth = encoding::columnWidthWithTabs(
839 Text.substr(LastBreak + 1), MultiLineString->OriginalColumn,
843 MultiLineString->ColumnWidth = encoding::columnWidthWithTabs(
844 FirstLineText, MultiLineString->OriginalColumn, Style.TabWidth, Encoding);
848 FormatToken *Tok = Tokens.back();
851 if (Tok->isNot(tok::numeric_constant))
853 StringRef Text = Tok->TokenText;
856 // 1. It starts with '+', '-'.
858 // 3. The first non-digit character is 'b', and the next is '0' or '1'.
859 // 4. The first non-digit character is 'x', and the next is a hex digit.
862 if (Text.size() < 1 || Text[0] == '+' || Text[0] == '-')
869 if (NonDigitPos < Text.size() - 1) {
880 Tok->Tok.setKind(tok::identifier);
881 Tok->Tok.setIdentifierInfo(nullptr);
886 FormatToken *BacktickToken = Tokens.back();
888 if (BacktickToken->is(tok::l_brace)) {
892 if (BacktickToken->is(tok::r_brace)) {
898 // If back in TEMPLATE_STRING, fallthrough and continue parsing the
899 } else if (BacktickToken->is(tok::unknown) &&
900 BacktickToken->TokenText == "`") {
907 const char *Offset = Lex->getBufferLocation();
908 const char *TmplBegin = Offset - BacktickToken->TokenText.size(); // at "`"
909 for (; Offset != Lex->getBuffer().end(); ++Offset) {
917 } else if (Offset + 1 < Lex->getBuffer().end() && Offset[0] == '$' &&
926 StringRef LiteralText(TmplBegin, Offset - TmplBegin);
927 BacktickToken->setType(TT_TemplateString);
928 BacktickToken->Tok.setKind(tok::string_literal);
929 BacktickToken->TokenText = LiteralText;
936 BacktickToken->ColumnWidth = encoding::columnWidthWithTabs(
937 FirstLineText, BacktickToken->OriginalColumn, Style.TabWidth, Encoding);
940 BacktickToken->IsMultiline = true;
942 BacktickToken->LastLineColumnWidth =
947 SourceLocation loc = Lex->getSourceLocation(Offset);
952 FormatToken *HashToken = Tokens.back();
953 if (!HashToken->isOneOf(tok::hash, tok::hashhash))
957 Lex->getBufferLocation() - HashToken->TokenText.size(); // at "#"
958 size_t From = CommentBegin - Lex->getBuffer().begin();
959 size_t To = Lex->getBuffer().find_first_of('\n', From);
961 To = Lex->getBuffer().size();
962 size_t Len = To - From;
963 HashToken->setType(TT_LineComment);
964 HashToken->Tok.setKind(tok::comment);
965 HashToken->TokenText = Lex->getBuffer().substr(From, Len);
966 SourceLocation Loc = To < Lex->getBuffer().size()
967 ? Lex->getSourceLocation(CommentBegin + Len)
975 FormatToken *Last = Tokens.back();
976 if (Last->isNot(tok::r_paren))
979 FormatToken *String = Tokens[Tokens.size() - 2];
980 if (String->isNot(tok::string_literal) || String->IsMultiline)
983 if (Tokens[Tokens.size() - 3]->isNot(tok::l_paren))
986 FormatToken *Macro = Tokens[Tokens.size() - 4];
987 if (Macro->TokenText != "_T")
990 const char *Start = Macro->TokenText.data();
991 const char *End = Last->TokenText.data() + Last->TokenText.size();
992 String->TokenText = StringRef(Start, End - Start);
993 String->IsFirst = Macro->IsFirst;
994 String->LastNewlineOffset = Macro->LastNewlineOffset;
995 String->WhitespaceRange = Macro->WhitespaceRange;
996 String->OriginalColumn = Macro->OriginalColumn;
997 String->ColumnWidth = encoding::columnWidthWithTabs(
998 String->TokenText, String->OriginalColumn, Style.TabWidth, Encoding);
999 String->NewlinesBefore = Macro->NewlinesBefore;
1000 String->HasUnescapedNewline = Macro->HasUnescapedNewline;
1005 Tokens.back() = String;
1007 FirstInLineIndex = Tokens.size() - 1;
1012 if (Tokens.back()->NewlinesBefore == 0 && Tokens.back()->isNot(tok::eof))
1028 Tokens[FirstInLineIndex]->getStartOfNonWhitespace());
1042 LineStart = Buffer.substr(LineOffset, FirstSpace - LineOffset);
1055 FormatToken *Next = Tokens.back();
1061 Tokens.back()->setType(Type);
1062 Tokens.back()->Tok.setKind(tok::kw___unknown_anytype);
1073 Token Tok = FormatTok->Tok;
1074 StringRef TokenText = FormatTok->TokenText;
1076 unsigned OriginalColumn = FormatTok->OriginalColumn;
1078 FormatTok->Tok = Tok;
1080 FormatTok->Tok.getLocation().getLocWithOffset(Tok.getLength() - 1);
1081 FormatTok->Tok.setLocation(TokLocation);
1082 FormatTok->WhitespaceRange = SourceRange(TokLocation, TokLocation);
1083 FormatTok->TokenText = TokenText;
1084 FormatTok->ColumnWidth = 1;
1085 FormatTok->OriginalColumn = OriginalColumn + 1;
1091 /// from the end of the truncated token. Used for other languages that have
1094 /// used to set the lexer to the end of whitespace if the lexer regards
1097 assert(NewLen <= FormatTok->TokenText.size());
1098 resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(
1099 Lex->getBufferLocation() - FormatTok->TokenText.size() + NewLen)));
1100 FormatTok->TokenText = FormatTok->TokenText.substr(0, NewLen);
1101 FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
1102 FormatTok->TokenText, FormatTok->OriginalColumn, Style.TabWidth,
1104 FormatTok->Tok.setLength(NewLen);
1115 const unsigned char *const End = Text.bytes_end();
1117 while (Cur < End) {
1123 // The source has a null byte at the end. So the end of the entire input
1126 assert(End - Cur >= 2);
1133 assert(End - Cur >= 4);
1139 return Cur - Begin;
1151 FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
1152 FormatTok->IsFirst = IsFirstToken;
1160 while (FormatTok->isNot(tok::eof)) {
1161 auto LeadingWhitespace = countLeadingWhitespace(FormatTok->TokenText);
1164 if (LeadingWhitespace < FormatTok->TokenText.size())
1166 StringRef Text = FormatTok->TokenText;
1178 ++FormatTok->NewlinesBefore;
1180 FormatTok->HasUnescapedNewline = true;
1183 FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
1195 Style.TabWidth - (Style.TabWidth ? Column % Style.TabWidth : 0);
1205 (i >= 1 && (Text.substr(i - 1, 4) == "\?\?/\r" ||
1206 Text.substr(i - 1, 4) == "\?\?/\n")) ||
1207 (i >= 2 && (Text.substr(i - 2, 4) == "\?\?/\r" ||
1208 Text.substr(i - 2, 4) == "\?\?/\n")));
1221 if (FormatTok->is(tok::unknown))
1222 FormatTok->setType(TT_ImplicitStringLiteral);
1224 // JavaScript and Java do not allow to escape the end of the line with a
1232 FormatTok->is(tok::comment) && FormatTok->TokenText.starts_with("//")) {
1233 size_t BackslashPos = FormatTok->TokenText.find('\\');
1235 if (BackslashPos + 1 < FormatTok->TokenText.size() &&
1236 FormatTok->TokenText[BackslashPos + 1] == '\n') {
1240 BackslashPos = FormatTok->TokenText.find('\\', BackslashPos + 1);
1251 // Verilog-specific stuff, the hash becomes an identifier.
1252 if (FormatTok->is(tok::numeric_constant)) {
1254 auto Quote = FormatTok->TokenText.find('\'');
1257 } else if (FormatTok->isOneOf(tok::hash, tok::hashhash)) {
1258 FormatTok->Tok.setKind(tok::raw_identifier);
1259 } else if (FormatTok->is(tok::raw_identifier)) {
1260 if (FormatTok->TokenText == "`") {
1261 FormatTok->Tok.setIdentifierInfo(nullptr);
1262 FormatTok->Tok.setKind(tok::hash);
1263 } else if (FormatTok->TokenText == "``") {
1264 FormatTok->Tok.setIdentifierInfo(nullptr);
1265 FormatTok->Tok.setKind(tok::hashhash);
1267 Tokens.back()->is(Keywords.kw_apostrophe) &&
1268 NumberBase.match(FormatTok->TokenText, &Matches)) {
1274 FormatTok->setFinalizedType(TT_VerilogNumberBase);
1279 FormatTok->WhitespaceRange = SourceRange(
1282 FormatTok->OriginalColumn = Column;
1285 if (FormatTok->is(tok::comment)) {
1287 StringRef UntrimmedText = FormatTok->TokenText;
1288 FormatTok->TokenText = FormatTok->TokenText.rtrim(" \t\v\f");
1289 TrailingWhitespace = UntrimmedText.size() - FormatTok->TokenText.size();
1290 } else if (FormatTok->is(tok::raw_identifier)) {
1291 IdentifierInfo &Info = IdentTable.get(FormatTok->TokenText);
1292 FormatTok->Tok.setIdentifierInfo(&Info);
1293 FormatTok->Tok.setKind(Info.getTokenID());
1295 FormatTok->isOneOf(tok::kw_struct, tok::kw_union, tok::kw_delete,
1297 FormatTok->Tok.setKind(tok::identifier);
1298 FormatTok->Tok.setIdentifierInfo(nullptr);
1300 FormatTok->isOneOf(tok::kw_struct, tok::kw_union,
1302 FormatTok->Tok.setKind(tok::identifier);
1303 FormatTok->Tok.setIdentifierInfo(nullptr);
1305 FormatTok->Tok.setKind(tok::identifier);
1306 FormatTok->Tok.setIdentifierInfo(nullptr);
1308 } else if (FormatTok->is(tok::greatergreater)) {
1309 FormatTok->Tok.setKind(tok::greater);
1310 FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
1313 } else if (FormatTok->is(tok::lessless)) {
1314 FormatTok->Tok.setKind(tok::less);
1315 FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
1321 Tokens.back()->is(TT_VerilogNumberBase) &&
1322 FormatTok->Tok.isOneOf(tok::identifier, tok::question)) {
1324 FormatTok->Tok.setKind(tok::numeric_constant);
1327 // Now FormatTok is the next non-whitespace token.
1329 StringRef Text = FormatTok->TokenText;
1334 FormatTok->ColumnWidth =
1336 Column += FormatTok->ColumnWidth;
1338 FormatTok->IsMultiline = true;
1341 FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
1346 FormatTok->LastLineColumnWidth = encoding::columnWidthWithTabs(
1348 Column = FormatTok->LastLineColumnWidth;
1352 auto *Identifier = FormatTok->Tok.getIdentifierInfo();
1354 if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() &&
1355 Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() ==
1357 it != Macros.end()) {
1358 FormatTok->setType(it->second);
1359 if (it->second == TT_IfMacro) {
1364 FormatTok->Tok.setKind(tok::kw_if);
1366 } else if (FormatTok->is(tok::identifier)) {
1368 FormatTok->setType(TT_MacroBlockBegin);
1370 FormatTok->setType(TT_MacroBlockEnd);
1372 FormatTok->setFinalizedType(TT_TypeName);
1398 const char *Start = Lex->getBufferLocation();
1399 if (!VerilogToken.match(StringRef(Start, Lex->getBuffer().end() - Start),
1403 // There is a null byte at the end of the buffer, so we don't have to check
1414 Tok.setLocation(Lex->getSourceLocation(Start, Len));
1416 Lex->seek(Lex->getCurrentBufferOffset() + Len, /*IsAtStartofline=*/false);
1421 // For Verilog, first see if there is a special token, and fall back to the
1424 Lex->LexFromRawLexer(Tok.Tok);
1453 Buffer.begin(), Buffer.begin() + Offset, Buffer.end()));
1454 Lex->SetKeepWhitespaceMode(true);