Lines Matching defs:tokens
108 TokenSequence tokens{dir, allSources_.AddCompilerInsertion(dir).start()};
109 tokens.Emit(cooked_);
114 TokenSequence tokens;
169 LabelField(tokens);
175 EmitChar(tokens, '!');
179 EmitChar(tokens, *sp);
182 EmitChar(tokens, ' ');
187 tokens.CloseToken();
244 LabelField(tokens);
249 while (NextToken(tokens)) {
262 preprocessor_.MacroReplacement(tokens, *this)}) {
315 while (CompilerDirectiveContinuation(tokens, line.sentinel)) {
318 tokens.ToLowerCase();
319 SourceFormChange(tokens.ToString());
321 tokens.ToLowerCase();
323 EnforceStupidEndStatementRules(tokens);
326 CheckAndEmitLine(tokens, newlineProvenance);
332 TokenSequence &tokens, Provenance newlineProvenance) {
333 tokens.CheckBadFortranCharacters(
341 !afterPreprocessingDirective_ && tokens.BadlyNestedParentheses() &&
346 tokens.CheckBadParentheses(messages_);
349 tokens.Emit(cooked_);
362 TokenSequence tokens;
363 while (NextToken(tokens)) {
366 return tokens;
445 void Prescanner::EnforceStupidEndStatementRules(const TokenSequence &tokens) {
446 CharBlock cBlock{tokens.ToCharBlock()};
459 auto start{allSources_.GetSourcePosition(tokens.GetCharProvenance(j))};
460 auto end{allSources_.GetSourcePosition(tokens.GetCharProvenance(n - 1))};
484 auto range{tokens.GetTokenProvenanceRange(1)};
490 allSources_.GetSourcePosition(tokens.GetCharProvenance(endOfPrefix))};
491 auto next{allSources_.GetSourcePosition(tokens.GetCharProvenance(j))};
630 bool Prescanner::NextToken(TokenSequence &tokens) {
664 tokens.PutNextTokenChar(' ', GetProvenance(theSpace));
665 tokens.CloseToken();
671 tokens.PutNextTokenChar(' ', spaceProvenance_);
679 QuotedCharacterLiteral(tokens, start);
688 EmitCharAndAdvance(tokens, *at_);
696 Hollerith(tokens, n, start);
698 while (IsDecimalDigit(EmitCharAndAdvance(tokens, *at_))) {
700 ExponentAndKind(tokens);
701 } else if (ExponentAndKind(tokens)) {
705 EmitCharAndAdvance(tokens, *at_);
708 EmitCharAndAdvance(tokens, *at_);
709 QuotedCharacterLiteral(tokens, start);
715 EmitCharAndAdvance(tokens, *at_);
719 char nch{EmitCharAndAdvance(tokens, '.')};
721 while (IsDecimalDigit(EmitCharAndAdvance(tokens, *at_))) {
723 ExponentAndKind(tokens);
724 } else if (nch == '.' && EmitCharAndAdvance(tokens, '.') == '.') {
725 EmitCharAndAdvance(tokens, '.'); // variadic macro definition ellipsis
732 EmitChar(tokens, *at_);
736 tokens.CloseToken();
744 // distinct pp-tokens so that macro replacement operates on them
758 tokens.ReopenLastToken();
765 tokens.CharAt(tokens.SizeInChars() - 1) == '_') { // kind_"..."
766 QuotedCharacterLiteral(tokens, start);
772 if (EmitCharAndAdvance(tokens, '*') == '*') {
773 EmitCharAndAdvance(tokens, '*');
784 isPossibleMacroCall_ = tokens.SizeInTokens() > 0 &&
786 tokens.TokenAt(tokens.SizeInTokens() - 1));
791 char nch{EmitCharAndAdvance(tokens, ch)};
800 EmitCharAndAdvance(tokens, nch);
815 tokens.CloseToken();
819 bool Prescanner::ExponentAndKind(TokenSequence &tokens) {
836 EmitCharAndAdvance(tokens, ed);
838 EmitCharAndAdvance(tokens, *at_);
841 EmitCharAndAdvance(tokens, *at_);
844 while (IsLegalInIdentifier(EmitCharAndAdvance(tokens, *at_))) {
854 TokenSequence &tokens, const char *start) {
859 const auto emit{[&](char ch) { EmitChar(tokens, ch); }};
860 const auto insert{[&](char ch) { EmitInsertedChar(tokens, ch); }};
877 while (PadOutCharacterLiteral(tokens)) {
911 tokens.Put(*replaced);
927 EmitChar(tokens, quote);
944 TokenSequence &tokens, int count, const char *start) {
947 EmitChar(tokens, 'H');
949 if (PadOutCharacterLiteral(tokens)) {
968 EmitChar(tokens, utf8.buffer[j]);
986 bool Prescanner::PadOutCharacterLiteral(TokenSequence &tokens) {
989 tokens.PutNextTokenChar(' ', spaceProvenance_);
1580 TokenSequence &tokens, Provenance newlineProvenance) const {
1582 tokens.PutNextTokenChar('\n', newlineProvenance);
1583 tokens.CloseToken();
1584 const char *ppd{tokens.ToCharBlock().begin()};
1586 tokens.pop_back(); // remove the newline
1599 // the tokens that constitute a compiler directive, even when those
1605 TokenSequence &tokens, const char *origSentinel) {
1606 if (inFixedForm_ || tokens.empty() ||
1607 tokens.TokenAt(tokens.SizeInTokens() - 1) != "&") {
1669 tokens.pop_back(); // delete original '&'
1670 tokens.Put(followingTokens, startAt, following - startAt);
1671 tokens.RemoveRedundantBlanks();
1679 bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
1680 if (!inFixedForm_ && !tokens.empty() &&
1681 tokens.TokenAt(tokens.SizeInTokens() - 1) == "&") {
1699 tokens.pop_back(); // delete original '&'
1700 tokens.Put(followingTokens);