Lines Matching +full:- +full:- +full:token
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
25 #include "clang/Lex/Token.h"
39 void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
54 Tokens = &*Macro->tokens_begin();
58 NumTokens = Macro->tokens_end()-Macro->tokens_begin();
73 // creating separate source location entries for each token.
75 MacroDefLength = Macro->getDefinitionLength(SM);
82 // If this is a function-like macro, expand the arguments and change
84 if (Macro->isFunctionLike() && Macro->getNumParams())
88 // expanded. The macro must be disabled only after argument pre-expansion of
89 // function-like macro arguments occurs.
90 Macro->DisableMacro();
93 /// Create a TokenLexer for the specified token stream. This does not
94 /// take ownership of the specified token vector.
95 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
117 // Set HasLeadingSpace/AtStartOfLine so that the first token will be
126 // If this was a function-like macro that actually uses its arguments, delete
135 if (ActualArgs) ActualArgs->destroy(PP);
139 SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
142 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
145 // In Microsoft-compatibility mode, a comma is removed in the expansion
157 && Macro->getNumParams() < 2)
173 // then removal of the comma should produce a placemarker token (in C99
180 ResultToks.back().setFlag(Token::CommaAfterElided);
189 SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
192 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
193 Token *const VAOPTTokens =
196 SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
198 // encounter pasting - and only then perform this loop.
200 // Perform token pasting (concatenation) prior to stringization.
206 Token &LHS = VAOPTTokens[CurTokenIdx - 1];
209 // Replace the token prior to the first ## in this iteration.
221 // macro definition, and use it to indicate that the stringified token
228 Token StringifiedVAOPT = MacroArgs::StringifyArgument(
233 StringifiedVAOPT.setFlag(Token::LeadingSpace);
235 StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
236 // Resize (shrink) the token stream to just capture this stringified token.
241 /// Expand the arguments of a function-like macro so that we can quickly
244 SmallVector<Token, 128> ResultToks;
256 const Token &CurTok = Tokens[I];
257 // We don't want a space for the next token after a paste
258 // operator. In valid code, the token will get smooshed onto the
259 // preceding one anyway. In assembler-with-cpp mode, invalid
262 // -> ".foo" not ". foo".
263 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
281 // If we are about to process a token that is either an argument to
283 // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
285 // and/or skip to the next token.
287 // token.
288 // 3) else (macro was invoked with variadic arguments) process the token
295 // this token. Note sawClosingParen() returns true only if the r_paren matches
301 ActualArgs->invokedWithVariadicArgument(Macro, PP);
304 // Skip this token.
308 // have a closing rparen - so process this token normally.
310 // Current token is the closing r_paren which marks the end of the
311 // __VA_OPT__ invocation, so handle any place-marker pasting (if
314 // but do so only after any token concatenation that needs to occur
319 // stringified token. This requires token-pasting to eagerly occur
322 // is a token that represents an empty string.
328 // Treat VAOPT as a placemarker token. Eat either the '##' before the
346 ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
348 "no token paste before __VA_OPT__");
350 VCtx.getNumberOfTokensPriorToVAOpt() - 1);
353 // following '##' token.
361 // __VA_OPT__), so skip to the next token.
367 // preprocessor already verified that the following token is a macro
371 int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
372 assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
373 "Token following # is not an argument or __VA_OPT__!");
375 if (ArgNo == -1) {
388 const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);
389 Token Res = MacroArgs::StringifyArgument(
391 Res.setFlag(Token::StringifiedInMacro);
396 Res.setFlag(Token::LeadingSpace);
405 // Find out if there is a paste (##) operator before or after the token.
408 bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
415 // Otherwise, if this is not an argument token, just add the token to the
418 int ArgNo = II ? Macro->getParameterNum(II) : -1;
419 if (ArgNo == -1) {
424 ResultToks.back().setFlag(Token::LeadingSpace);
427 ResultToks.back().clearFlag(Token::LeadingSpace);
440 if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
446 // If it is not the LHS/RHS of a ## operator, we must pre-expand the
450 const Token *ResultArgToks;
454 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
455 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
456 ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
458 ResultArgToks = ArgTok; // Use non-preexpanded tokens.
460 // If the arg token expanded into anything, append it.
461 if (ResultArgToks->isNot(tok::eof)) {
466 // In Microsoft-compatibility mode, we follow MSVC's preprocessing
468 // expansions as argument separators. Set a flag on the token so we can
472 ResultToks.back().setFlag(Token::IgnoredComma);
476 for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))
487 // before the first token should match the whitespace of the arg
489 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
491 ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
494 // We're creating a placeholder token. Usually this doesn't matter,
498 // We're imagining a placeholder token is inserted here. If this is
499 // the first token in a __VA_OPT__ after a ##, delete the ##.
508 // Okay, we have a token that is either the LHS or RHS of a paste (##)
509 // argument. It gets substituted as its non-pre-expanded tokens.
510 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
516 // the expander tries to paste ',' with the first token of the __VA_ARGS__
519 ResultToks[ResultToks.size()-2].is(tok::comma) &&
520 (unsigned)ArgNo == Macro->getNumParams()-1 &&
521 Macro->isVariadic()) {
531 for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
539 ResultToks.end()-NumToks, ResultToks.end());
542 // Transfer the leading whitespace information from the token
543 // (the macro argument) onto the first token of the
545 // pseudo-paste extension ", ## __VA_ARGS__".
547 ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
549 ResultToks[ResultToks.size() - NumToks].setFlagValue(
550 Token::LeadingSpace, NextTokGetsSpace);
562 // Discard the argument token and skip (don't copy to the expansion
588 // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
589 // and if the macro had at least one real argument, and if the token before
591 // disabled when using -std=c99.
592 if (ActualArgs->isVarargsElidedUse())
600 assert(!OwnsTokens && "This would leak if we already own the token list");
613 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
614 const Token &SecondTok) {
616 FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
620 /// Lex - Lex and return a token from this macro stream.
621 bool TokenLexer::Lex(Token &Tok) {
624 // If this is a macro (not a token stream), mark the macro enabled now
626 if (Macro) Macro->EnableMacro();
629 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
630 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
632 Tok.setFlag(Token::LeadingEmptyMacro);
638 // If this is the first token of the expanded result, we inherit spacing
642 // Get the next token to return.
645 Tok.setFlag(Token::IsReinjected);
649 // If this token is followed by a token paste (##) operator, paste the tokens!
650 // Note that ## is a normal token when not expanding a macro.
653 // Special processing of L#x macros in -fms-compatibility mode.
655 // 'L#macro_arg' construct in a function-like macro.
658 // When handling the microsoft /##/ extension, the final token is
659 // returned by pasteTokens, not the pasted token.
666 // The token's current location indicate where the token was lexed from. We
667 // need this information to compute the spelling of the token, but any
668 // diagnostics for the expanded token should appear as if they came from
671 if (ExpandLocStart.isValid() && // Don't do this for token streams.
672 // Check that the token's location was not already set properly.
687 // If this is the first token, set the lexical properties of the token to
690 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
691 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
693 // If this is not the first token, we may still need to pass through
695 if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
696 if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
703 // Change the kind of this identifier to the appropriate token kind, e.g.
706 Tok.setKind(II->getTokenID());
711 if (II->isPoisoned() && TokenIsFromPaste) {
715 if (!DisableMacroExpansion && II->isHandleIdentifierCase())
719 // Otherwise, return a normal token.
723 bool TokenLexer::pasteTokens(Token &Tok) {
730 /// If this returns true, the caller should immediately return the token.
731 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
733 assert(CurIdx > 0 && "## can not be the first token within tokens");
737 "Token at this Index must be ## or part of the MSVC 'L "
738 "#macro-arg' pasting pair");
740 // MSVC: If previous token was pasted, this must be a recovery from an invalid
741 // paste operation. Ignore spaces before this token to mimic MSVC output.
744 TokenStream[CurIdx - 2].is(tok::hashhash))
745 LHSTok.clearFlag(Token::LeadingSpace);
761 assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
763 // Get the RHS token.
764 const Token &RHS = TokenStream[CurIdx];
766 // Allocate space for the result token. This is guaranteed to be enough for
770 // Get the spelling of the LHS token in Buffer.
792 Token ResultTokTmp;
795 // Claim that the tmp token is a string_literal so that we can get the
802 // Lex the resultant pasted token into Result.
803 Token Result;
828 // Make a lexer to lex this string from. Lex just this one token.
834 // Lex a token in raw mode. This way it won't look up identifiers
835 // automatically, lexing off the end will return an eof token, and
836 // warnings are disabled. This returns true if the result token is the
840 // If we got an EOF token, we didn't form even ONE token. For example, we
844 // If pasting the two tokens didn't form a full new token, this is an
846 // unmodified and with RHS as the next token to lex.
848 // Explicitly convert the token location to have proper expansion
883 Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());
884 Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());
891 SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
893 // The token's current location indicate where the token was lexed from. We
894 // need this information to compute the spelling of the token, but any
895 // diagnostics for the expanded token should appear as if the token was
912 // Now that we got the result token, it will be subject to expansion. Since
913 // token pasting re-lexes the result token in raw mode, identifier information
916 // Look up the identifier info for the token. We disabled identifier lookup
923 /// isNextTokenLParen - If the next token lexed will pop this macro off the
924 /// expansion stack, return 2. If the next unexpanded token is a '(', return
933 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
936 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
939 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
943 /// first token on the next line.
944 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
952 assert(Macro && "Token streams can't paste comments");
953 Macro->EnableMacro();
965 "Not appropriate for token streams");
978 /// creates a single SLocEntry, and assigns SourceLocations to each token that
988 Token *&begin_tokens,
989 Token * end_tokens) {
991 SourceLocation BeginLoc = begin_tokens->getLocation();
992 llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens);
993 llvm::MutableArrayRef<Token> Partition;
1000 auto Distance = Loc.getRawEncoding() - Last.getRawEncoding();
1011 Partition = All.take_while([&](const Token &T) {
1016 // sourcelocation-against-bounds comparison.
1020 Partition = All.take_while([&](const Token &T) {
1022 // single token past the end of the FileID, specifically the ) when a
1023 // macro-arg containing a comma should be guarded by parentheses.
1028 // See https://github.com/llvm/llvm-project/issues/60722.
1038 Partition.back().getEndLoc().getRawEncoding() -
1047 const Token &T) {
1054 for (Token& T : Partition) {
1056 T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding();
1068 Token *begin_tokens,
1069 Token *end_tokens) {
1076 // If there's only one token just create a SLocEntry for it.
1077 if (end_tokens - begin_tokens == 1) {
1078 Token &Tok = *begin_tokens;
1089 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {