xref: /llvm-project/flang/lib/Parser/preprocessor.cpp (revision 850d42fb145c636a3b56a7616c3e3c5c188c1916) 
1 //===-- lib/Parser/preprocessor.cpp ---------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "flang/Parser/preprocessor.h"
10 
11 #include "prescan.h"
12 #include "flang/Common/idioms.h"
13 #include "flang/Parser/characters.h"
14 #include "flang/Parser/message.h"
15 #include "llvm/Support/FileSystem.h"
16 #include "llvm/Support/raw_ostream.h"
17 #include <algorithm>
18 #include <cinttypes>
19 #include <cstddef>
20 #include <ctime>
21 #include <map>
22 #include <memory>
23 #include <optional>
24 #include <set>
25 #include <utility>
26 #include <vector>
27 
28 namespace Fortran::parser {
29 
30 Definition::Definition(
31     const TokenSequence &repl, std::size_t firstToken, std::size_t tokens)
32     : replacement_{Tokenize({}, repl, firstToken, tokens)} {}
33 
34 Definition::Definition(const std::vector<std::string> &argNames,
35     const TokenSequence &repl, std::size_t firstToken, std::size_t tokens,
36     bool isVariadic)
37     : isFunctionLike_{true}, isVariadic_{isVariadic}, argNames_{argNames},
38       replacement_{Tokenize(argNames, repl, firstToken, tokens)} {}
39 
40 Definition::Definition(const std::string &predefined, AllSources &sources)
41     : isPredefined_{true},
42       replacement_{
43           predefined, sources.AddCompilerInsertion(predefined).start()} {}
44 
45 bool Definition::set_isDisabled(bool disable) {
46   bool was{isDisabled_};
47   isDisabled_ = disable;
48   return was;
49 }
50 
51 void Definition::Print(llvm::raw_ostream &out, const char *macroName) const {
52   if (!isFunctionLike_) {
53     // If it's not a function-like macro, then just print the replacement.
54     out << ' ' << replacement_.ToString();
55     return;
56   }
57 
58   size_t argCount{argumentCount()};
59 
60   out << '(';
61   for (size_t i{0}; i != argCount; ++i) {
62     if (i != 0) {
63       out << ", ";
64     }
65     out << argNames_[i];
66   }
67   if (isVariadic_) {
68     out << ", ...";
69   }
70   out << ") ";
71 
72   for (size_t i{0}, e{replacement_.SizeInTokens()}; i != e; ++i) {
73     std::string tok{replacement_.TokenAt(i).ToString()};
74     if (size_t idx{GetArgumentIndex(tok)}; idx < argCount) {
75       out << argNames_[idx];
76     } else {
77       out << tok;
78     }
79   }
80 }
81 
82 static bool IsLegalIdentifierStart(const CharBlock &cpl) {
83   return cpl.size() > 0 && IsLegalIdentifierStart(cpl[0]);
84 }
85 
86 TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
87     const TokenSequence &token, std::size_t firstToken, std::size_t tokens) {
88   std::map<std::string, std::string> args;
89   char argIndex{'A'};
90   for (const std::string &arg : argNames) {
91     CHECK(args.find(arg) == args.end());
92     args[arg] = "~"s + argIndex++;
93   }
94   TokenSequence result;
95   for (std::size_t j{0}; j < tokens; ++j) {
96     CharBlock tok{token.TokenAt(firstToken + j)};
97     if (IsLegalIdentifierStart(tok)) {
98       auto it{args.find(tok.ToString())};
99       if (it != args.end()) {
100         result.Put(it->second, token.GetTokenProvenance(j));
101         continue;
102       }
103     }
104     result.Put(token, firstToken + j, 1);
105   }
106   return result;
107 }
108 
109 std::size_t Definition::GetArgumentIndex(const CharBlock &token) const {
110   if (token.size() >= 2 && token[0] == '~') {
111     return static_cast<size_t>(token[1] - 'A');
112   }
113   return argumentCount();
114 }
115 
116 static TokenSequence Stringify(
117     const TokenSequence &tokens, AllSources &allSources) {
118   TokenSequence result;
119   Provenance quoteProvenance{allSources.CompilerInsertionProvenance('"')};
120   result.PutNextTokenChar('"', quoteProvenance);
121   for (std::size_t j{0}; j < tokens.SizeInTokens(); ++j) {
122     const CharBlock &token{tokens.TokenAt(j)};
123     std::size_t bytes{token.size()};
124     for (std::size_t k{0}; k < bytes; ++k) {
125       char ch{token[k]};
126       Provenance from{tokens.GetTokenProvenance(j, k)};
127       if (ch == '"' || ch == '\\') {
128         result.PutNextTokenChar(ch, from);
129       }
130       result.PutNextTokenChar(ch, from);
131     }
132   }
133   result.PutNextTokenChar('"', quoteProvenance);
134   result.CloseToken();
135   return result;
136 }
137 
138 constexpr bool IsTokenPasting(CharBlock opr) {
139   return opr.size() == 2 && opr[0] == '#' && opr[1] == '#';
140 }
141 
142 static bool AnyTokenPasting(const TokenSequence &text) {
143   std::size_t tokens{text.SizeInTokens()};
144   for (std::size_t j{0}; j < tokens; ++j) {
145     if (IsTokenPasting(text.TokenAt(j))) {
146       return true;
147     }
148   }
149   return false;
150 }
151 
152 static TokenSequence TokenPasting(TokenSequence &&text) {
153   if (!AnyTokenPasting(text)) {
154     return std::move(text);
155   }
156   TokenSequence result;
157   std::size_t tokens{text.SizeInTokens()};
158   bool pasting{false};
159   for (std::size_t j{0}; j < tokens; ++j) {
160     if (IsTokenPasting(text.TokenAt(j))) {
161       if (!pasting) {
162         while (!result.empty() &&
163             result.TokenAt(result.SizeInTokens() - 1).IsBlank()) {
164           result.pop_back();
165         }
166         if (!result.empty()) {
167           result.ReopenLastToken();
168           pasting = true;
169         }
170       }
171     } else if (pasting && text.TokenAt(j).IsBlank()) {
172     } else {
173       result.Put(text, j, 1);
174       pasting = false;
175     }
176   }
177   return result;
178 }
179 
180 constexpr bool IsDefinedKeyword(CharBlock token) {
181   return token.size() == 7 && (token[0] == 'd' || token[0] == 'D') &&
182       ToLowerCaseLetters(token.ToString()) == "defined";
183 }
184 
185 TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
186     Prescanner &prescanner, bool inIfExpression) {
187   TokenSequence result;
188   bool skipping{false};
189   int parenthesesNesting{0};
190   std::size_t tokens{replacement_.SizeInTokens()};
191   for (std::size_t j{0}; j < tokens; ++j) {
192     CharBlock token{replacement_.TokenAt(j)};
193     std::size_t bytes{token.size()};
194     if (skipping) {
195       char ch{token.OnlyNonBlank()};
196       if (ch == '(') {
197         ++parenthesesNesting;
198       } else if (ch == ')') {
199         if (parenthesesNesting > 0) {
200           --parenthesesNesting;
201         }
202         skipping = parenthesesNesting > 0;
203       }
204       continue;
205     }
206     if (bytes == 2 && token[0] == '~') { // argument substitution
207       std::size_t index{GetArgumentIndex(token)};
208       if (index >= args.size()) {
209         continue;
210       }
211       std::size_t prev{j};
212       while (prev > 0 && replacement_.TokenAt(prev - 1).IsBlank()) {
213         --prev;
214       }
215       if (prev > 0 && replacement_.TokenAt(prev - 1).size() == 1 &&
216           replacement_.TokenAt(prev - 1)[0] ==
217               '#') { // stringify argument without macro replacement
218         std::size_t resultSize{result.SizeInTokens()};
219         while (resultSize > 0 && result.TokenAt(resultSize - 1).IsBlank()) {
220           result.pop_back();
221           --resultSize;
222         }
223         CHECK(resultSize > 0 &&
224             result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
225         result.pop_back();
226         result.Put(Stringify(args[index], prescanner.allSources()));
227       } else {
228         const TokenSequence *arg{&args[index]};
229         std::optional<TokenSequence> replaced;
230         // Don't replace macros in the actual argument if it is preceded or
231         // followed by the token-pasting operator ## in the replacement text,
232         // or if we have to worry about "defined(X)"/"defined X" in an
233         // #if/#elif expression.
234         if (!inIfExpression &&
235             (prev == 0 || !IsTokenPasting(replacement_.TokenAt(prev - 1)))) {
236           auto next{replacement_.SkipBlanks(j + 1)};
237           if (next >= tokens || !IsTokenPasting(replacement_.TokenAt(next))) {
238             // Apply macro replacement to the actual argument
239             replaced = prescanner.preprocessor().MacroReplacement(
240                 *arg, prescanner, nullptr, inIfExpression);
241             if (replaced) {
242               arg = &*replaced;
243             }
244           }
245         }
246         result.Put(DEREF(arg));
247       }
248     } else if (bytes == 11 && isVariadic_ &&
249         token.ToString() == "__VA_ARGS__") {
250       Provenance commaProvenance{
251           prescanner.preprocessor().allSources().CompilerInsertionProvenance(
252               ',')};
253       for (std::size_t k{argumentCount()}; k < args.size(); ++k) {
254         if (k > argumentCount()) {
255           result.Put(","s, commaProvenance);
256         }
257         result.Put(args[k]);
258       }
259     } else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
260         j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
261         parenthesesNesting == 0) {
262       parenthesesNesting = 1;
263       skipping = args.size() == argumentCount();
264       ++j;
265     } else {
266       if (parenthesesNesting > 0) {
267         char ch{token.OnlyNonBlank()};
268         if (ch == '(') {
269           ++parenthesesNesting;
270         } else if (ch == ')') {
271           if (--parenthesesNesting == 0) {
272             skipping = false;
273             continue;
274           }
275         }
276       }
277       result.Put(replacement_, j);
278     }
279   }
280   return TokenPasting(std::move(result));
281 }
282 
283 static std::string FormatTime(const std::time_t &now, const char *format) {
284   char buffer[16];
285   return {buffer,
286       std::strftime(buffer, sizeof buffer, format, std::localtime(&now))};
287 }
288 
289 Preprocessor::Preprocessor(AllSources &allSources) : allSources_{allSources} {}
290 
291 void Preprocessor::DefineStandardMacros() {
292   // Capture current local date & time once now to avoid having the values
293   // of __DATE__ or __TIME__ change during compilation.
294   std::time_t now;
295   std::time(&now);
296   Define("__DATE__"s, FormatTime(now, "\"%h %e %Y\"")); // e.g., "Jun 16 1904"
297   Define("__TIME__"s, FormatTime(now, "\"%T\"")); // e.g., "23:59:60"
298   // The values of these predefined macros depend on their invocation sites.
299   Define("__FILE__"s, "__FILE__"s);
300   Define("__LINE__"s, "__LINE__"s);
301   Define("__TIMESTAMP__"s, "__TIMESTAMP__"s);
302 }
303 
304 void Preprocessor::Define(const std::string &macro, const std::string &value) {
305   definitions_.emplace(SaveTokenAsName(macro), Definition{value, allSources_});
306 }
307 
308 void Preprocessor::Undefine(std::string macro) { definitions_.erase(macro); }
309 
310 std::optional<TokenSequence> Preprocessor::MacroReplacement(
311     const TokenSequence &input, Prescanner &prescanner,
312     std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
313   // Do quick scan for any use of a defined name.
314   if (definitions_.empty()) {
315     return std::nullopt;
316   }
317   std::size_t tokens{input.SizeInTokens()};
318   std::size_t j{0};
319   for (; j < tokens; ++j) {
320     CharBlock token{input.TokenAt(j)};
321     if (!token.empty() && IsLegalIdentifierStart(token[0]) &&
322         (IsNameDefined(token) || (inIfExpression && IsDefinedKeyword(token)))) {
323       break;
324     }
325   }
326   if (j == tokens) {
327     return std::nullopt; // input contains nothing that would be replaced
328   }
329   TokenSequence result{input, 0, j};
330 
331   // After rescanning after macro replacement has failed due to an unclosed
332   // function-like macro call (no left parenthesis yet, or no closing
333   // parenthesis), if tokens remain in the input, append them to the
334   // replacement text and attempt to proceed.  Otherwise, return, so that
335   // the caller may try again with remaining tokens in its input.
336   auto CompleteFunctionLikeMacro{
337       [this, &input, &prescanner, &result, &partialFunctionLikeMacro,
338           inIfExpression](std::size_t after, const TokenSequence &replacement,
339           std::size_t pFLMOffset) {
340         if (after < input.SizeInTokens()) {
341           result.Put(replacement, 0, pFLMOffset);
342           TokenSequence suffix;
343           suffix.Put(
344               replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
345           suffix.Put(input, after, input.SizeInTokens() - after);
346           auto further{ReplaceMacros(
347               suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
348           if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
349             // still not closed
350             **partialFunctionLikeMacro += result.SizeInTokens();
351           }
352           result.Put(further);
353           return true;
354         } else {
355           if (partialFunctionLikeMacro) {
356             *partialFunctionLikeMacro = pFLMOffset + result.SizeInTokens();
357           }
358           return false;
359         }
360       }};
361 
362   for (; j < tokens; ++j) {
363     CharBlock token{input.TokenAt(j)};
364     if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
365       result.Put(input, j);
366       continue;
367     }
368     // Process identifier in replacement text.
369     auto it{definitions_.find(token)};
370     // Is in the X in "defined(X)" or "defined X" in an #if/#elif expression?
371     if (inIfExpression) {
372       if (auto prev{result.SkipBlanksBackwards(result.SizeInTokens())}) {
373         bool ok{true};
374         std::optional<std::size_t> rightParenthesis;
375         if (result.TokenAt(*prev).OnlyNonBlank() == '(') {
376           prev = result.SkipBlanksBackwards(*prev);
377           rightParenthesis = input.SkipBlanks(j + 1);
378           ok = *rightParenthesis < tokens &&
379               input.TokenAt(*rightParenthesis).OnlyNonBlank() == ')';
380         }
381         if (ok && prev && IsDefinedKeyword(result.TokenAt(*prev))) {
382           result = TokenSequence{result, 0, *prev}; // trims off "defined ("
383           char truth{it != definitions_.end() ? '1' : '0'};
384           result.Put(&truth, 1, allSources_.CompilerInsertionProvenance(truth));
385           j = rightParenthesis.value_or(j);
386           continue;
387         }
388       }
389     }
390     if (it == definitions_.end()) {
391       result.Put(input, j);
392       continue;
393     }
394     Definition *def{&it->second};
395     if (def->isDisabled()) {
396       result.Put(input, j);
397       continue;
398     }
399     if (!def->isFunctionLike()) {
400       if (def->isPredefined() && !def->replacement().empty()) {
401         std::string repl;
402         std::string name{def->replacement().TokenAt(0).ToString()};
403         if (name == "__FILE__") {
404           repl = "\""s +
405               allSources_.GetPath(prescanner.GetCurrentProvenance()) + '"';
406         } else if (name == "__LINE__") {
407           std::string buf;
408           llvm::raw_string_ostream ss{buf};
409           ss << allSources_.GetLineNumber(prescanner.GetCurrentProvenance());
410           repl = ss.str();
411         } else if (name == "__TIMESTAMP__") {
412           auto path{allSources_.GetPath(
413               prescanner.GetCurrentProvenance(), /*topLevel=*/true)};
414           llvm::sys::fs::file_status status;
415           repl = "??? ??? ?? ??:??:?? ????";
416           if (!llvm::sys::fs::status(path, status)) {
417             auto modTime{llvm::sys::toTimeT(status.getLastModificationTime())};
418             if (std::string time{std::asctime(std::localtime(&modTime))};
419                 time.size() > 1 && time[time.size() - 1] == '\n') {
420               time.erase(time.size() - 1); // clip terminal '\n'
421               repl = "\""s + time + '"';
422             }
423           }
424         }
425         if (!repl.empty()) {
426           ProvenanceRange insert{allSources_.AddCompilerInsertion(repl)};
427           ProvenanceRange call{allSources_.AddMacroCall(
428               insert, input.GetTokenProvenanceRange(j), repl)};
429           result.Put(repl, call.start());
430           continue;
431         }
432       }
433       std::optional<std::size_t> partialFLM;
434       def->set_isDisabled(true);
435       TokenSequence replaced{TokenPasting(ReplaceMacros(
436           def->replacement(), prescanner, &partialFLM, inIfExpression))};
437       def->set_isDisabled(false);
438       if (partialFLM &&
439           CompleteFunctionLikeMacro(j + 1, replaced, *partialFLM)) {
440         return result;
441       }
442       if (!replaced.empty()) {
443         ProvenanceRange from{def->replacement().GetProvenanceRange()};
444         ProvenanceRange use{input.GetTokenProvenanceRange(j)};
445         ProvenanceRange newRange{
446             allSources_.AddMacroCall(from, use, replaced.ToString())};
447         result.Put(replaced, newRange);
448       }
449     } else {
450       // Possible function-like macro call.  Skip spaces and newlines to see
451       // whether '(' is next.
452       std::size_t k{j};
453       bool leftParen{false};
454       while (++k < tokens) {
455         const CharBlock &lookAhead{input.TokenAt(k)};
456         if (!lookAhead.IsBlank() && lookAhead[0] != '\n') {
457           leftParen = lookAhead[0] == '(' && lookAhead.size() == 1;
458           break;
459         }
460       }
461       if (!leftParen) {
462         if (partialFunctionLikeMacro) {
463           *partialFunctionLikeMacro = result.SizeInTokens();
464           result.Put(input, j, tokens - j);
465           return result;
466         } else {
467           result.Put(input, j);
468           continue;
469         }
470       }
471       std::vector<std::size_t> argStart{++k};
472       for (int nesting{0}; k < tokens; ++k) {
473         CharBlock token{input.TokenAt(k)};
474         char ch{token.OnlyNonBlank()};
475         if (ch == '(') {
476           ++nesting;
477         } else if (ch == ')') {
478           if (nesting == 0) {
479             break;
480           }
481           --nesting;
482         } else if (ch == ',' && nesting == 0) {
483           argStart.push_back(k + 1);
484         }
485       }
486       if (argStart.size() == 1 && k == argStart[0] &&
487           def->argumentCount() == 0) {
488         // Subtle: () is zero arguments, not one empty argument,
489         // unless one argument was expected.
490         argStart.clear();
491       }
492       if (k >= tokens && partialFunctionLikeMacro) {
493         *partialFunctionLikeMacro = result.SizeInTokens();
494         result.Put(input, j, tokens - j);
495         return result;
496       } else if (k >= tokens || argStart.size() < def->argumentCount() ||
497           (argStart.size() > def->argumentCount() && !def->isVariadic())) {
498         result.Put(input, j);
499         continue;
500       }
501       std::vector<TokenSequence> args;
502       for (std::size_t n{0}; n < argStart.size(); ++n) {
503         std::size_t at{argStart[n]};
504         std::size_t count{
505             (n + 1 == argStart.size() ? k : argStart[n + 1] - 1) - at};
506         args.emplace_back(TokenSequence(input, at, count));
507       }
508       TokenSequence applied{def->Apply(args, prescanner, inIfExpression)};
509       std::optional<std::size_t> partialFLM;
510       def->set_isDisabled(true);
511       TokenSequence replaced{ReplaceMacros(
512           std::move(applied), prescanner, &partialFLM, inIfExpression)};
513       def->set_isDisabled(false);
514       if (partialFLM &&
515           CompleteFunctionLikeMacro(k + 1, replaced, *partialFLM)) {
516         return result;
517       }
518       if (!replaced.empty()) {
519         ProvenanceRange from{def->replacement().GetProvenanceRange()};
520         ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
521         ProvenanceRange newRange{
522             allSources_.AddMacroCall(from, use, replaced.ToString())};
523         result.Put(replaced, newRange);
524       }
525       j = k; // advance to the terminal ')'
526     }
527   }
528   return result;
529 }
530 
531 TokenSequence Preprocessor::ReplaceMacros(const TokenSequence &tokens,
532     Prescanner &prescanner,
533     std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
534   if (std::optional<TokenSequence> repl{MacroReplacement(
535           tokens, prescanner, partialFunctionLikeMacro, inIfExpression)}) {
536     return std::move(*repl);
537   }
538   return tokens;
539 }
540 
541 void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
542   std::size_t tokens{dir.SizeInTokens()};
543   std::size_t j{dir.SkipBlanks(0)};
544   if (j == tokens) {
545     return;
546   }
547   if (dir.TokenAt(j).ToString() != "#") {
548     prescanner.Say(dir.GetTokenProvenanceRange(j), "missing '#'"_err_en_US);
549     return;
550   }
551   j = dir.SkipBlanks(j + 1);
552   while (tokens > 0 && dir.TokenAt(tokens - 1).IsBlank()) {
553     --tokens;
554   }
555   if (j == tokens) {
556     return;
557   }
558   if (IsDecimalDigit(dir.TokenAt(j)[0]) || dir.TokenAt(j)[0] == '"') {
559     LineDirective(dir, j, prescanner);
560     return;
561   }
562   std::size_t dirOffset{j};
563   std::string dirName{ToLowerCaseLetters(dir.TokenAt(dirOffset).ToString())};
564   j = dir.SkipBlanks(j + 1);
565   CharBlock nameToken;
566   if (j < tokens && IsLegalIdentifierStart(dir.TokenAt(j)[0])) {
567     nameToken = dir.TokenAt(j);
568   }
569   if (dirName == "line") {
570     LineDirective(dir, j, prescanner);
571   } else if (dirName == "define") {
572     if (nameToken.empty()) {
573       prescanner.Say(dir.GetTokenProvenanceRange(j < tokens ? j : tokens - 1),
574           "#define: missing or invalid name"_err_en_US);
575       return;
576     }
577     nameToken = SaveTokenAsName(nameToken);
578     definitions_.erase(nameToken);
579     if (++j < tokens && dir.TokenAt(j).OnlyNonBlank() == '(') {
580       j = dir.SkipBlanks(j + 1);
581       std::vector<std::string> argName;
582       bool isVariadic{false};
583       if (dir.TokenAt(j).OnlyNonBlank() != ')') {
584         while (true) {
585           std::string an{dir.TokenAt(j).ToString()};
586           if (an == "...") {
587             isVariadic = true;
588           } else {
589             if (an.empty() || !IsLegalIdentifierStart(an[0])) {
590               prescanner.Say(dir.GetTokenProvenanceRange(j),
591                   "#define: missing or invalid argument name"_err_en_US);
592               return;
593             }
594             argName.push_back(an);
595           }
596           j = dir.SkipBlanks(j + 1);
597           if (j == tokens) {
598             prescanner.Say(dir.GetTokenProvenanceRange(tokens - 1),
599                 "#define: malformed argument list"_err_en_US);
600             return;
601           }
602           char punc{dir.TokenAt(j).OnlyNonBlank()};
603           if (punc == ')') {
604             break;
605           }
606           if (isVariadic || punc != ',') {
607             prescanner.Say(dir.GetTokenProvenanceRange(j),
608                 "#define: malformed argument list"_err_en_US);
609             return;
610           }
611           j = dir.SkipBlanks(j + 1);
612           if (j == tokens) {
613             prescanner.Say(dir.GetTokenProvenanceRange(tokens - 1),
614                 "#define: malformed argument list"_err_en_US);
615             return;
616           }
617         }
618         if (std::set<std::string>(argName.begin(), argName.end()).size() !=
619             argName.size()) {
620           prescanner.Say(dir.GetTokenProvenance(dirOffset),
621               "#define: argument names are not distinct"_err_en_US);
622           return;
623         }
624       }
625       j = dir.SkipBlanks(j + 1);
626       definitions_.emplace(std::make_pair(
627           nameToken, Definition{argName, dir, j, tokens - j, isVariadic}));
628     } else {
629       j = dir.SkipBlanks(j + 1);
630       definitions_.emplace(
631           std::make_pair(nameToken, Definition{dir, j, tokens - j}));
632     }
633   } else if (dirName == "undef") {
634     if (nameToken.empty()) {
635       prescanner.Say(
636           dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
637           "# missing or invalid name"_err_en_US);
638     } else {
639       if (dir.IsAnythingLeft(++j)) {
640         if (prescanner.features().ShouldWarn(
641                 common::UsageWarning::Portability)) {
642           prescanner.Say(common::UsageWarning::Portability,
643               dir.GetIntervalProvenanceRange(j, tokens - j),
644               "#undef: excess tokens at end of directive"_port_en_US);
645         }
646       } else {
647         definitions_.erase(nameToken);
648       }
649     }
650   } else if (dirName == "ifdef" || dirName == "ifndef") {
651     bool doThen{false};
652     if (nameToken.empty()) {
653       prescanner.Say(
654           dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
655           "#%s: missing name"_err_en_US, dirName);
656     } else {
657       if (dir.IsAnythingLeft(++j)) {
658         if (prescanner.features().ShouldWarn(
659                 common::UsageWarning::Portability)) {
660           prescanner.Say(common::UsageWarning::Portability,
661               dir.GetIntervalProvenanceRange(j, tokens - j),
662               "#%s: excess tokens at end of directive"_port_en_US, dirName);
663         }
664       }
665       doThen = IsNameDefined(nameToken) == (dirName == "ifdef");
666     }
667     if (doThen) {
668       ifStack_.push(CanDeadElseAppear::Yes);
669     } else {
670       SkipDisabledConditionalCode(dirName, IsElseActive::Yes, prescanner,
671           dir.GetTokenProvenance(dirOffset));
672     }
673   } else if (dirName == "if") {
674     if (IsIfPredicateTrue(dir, j, tokens - j, prescanner)) {
675       ifStack_.push(CanDeadElseAppear::Yes);
676     } else {
677       SkipDisabledConditionalCode(dirName, IsElseActive::Yes, prescanner,
678           dir.GetTokenProvenanceRange(dirOffset));
679     }
680   } else if (dirName == "else") {
681     if (dir.IsAnythingLeft(j)) {
682       if (prescanner.features().ShouldWarn(common::UsageWarning::Portability)) {
683         prescanner.Say(common::UsageWarning::Portability,
684             dir.GetIntervalProvenanceRange(j, tokens - j),
685             "#else: excess tokens at end of directive"_port_en_US);
686       }
687     } else if (ifStack_.empty()) {
688       prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
689           "#else: not nested within #if, #ifdef, or #ifndef"_err_en_US);
690     } else if (ifStack_.top() != CanDeadElseAppear::Yes) {
691       prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
692           "#else: already appeared within this #if, #ifdef, or #ifndef"_err_en_US);
693     } else {
694       ifStack_.pop();
695       SkipDisabledConditionalCode("else", IsElseActive::No, prescanner,
696           dir.GetTokenProvenanceRange(dirOffset));
697     }
698   } else if (dirName == "elif") {
699     if (ifStack_.empty()) {
700       prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
701           "#elif: not nested within #if, #ifdef, or #ifndef"_err_en_US);
702     } else if (ifStack_.top() != CanDeadElseAppear::Yes) {
703       prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
704           "#elif: #else previously appeared within this #if, #ifdef, or #ifndef"_err_en_US);
705     } else {
706       ifStack_.pop();
707       SkipDisabledConditionalCode("elif", IsElseActive::No, prescanner,
708           dir.GetTokenProvenanceRange(dirOffset));
709     }
710   } else if (dirName == "endif") {
711     if (dir.IsAnythingLeft(j)) {
712       if (prescanner.features().ShouldWarn(common::UsageWarning::Portability)) {
713         prescanner.Say(common::UsageWarning::Portability,
714             dir.GetIntervalProvenanceRange(j, tokens - j),
715             "#endif: excess tokens at end of directive"_port_en_US);
716       }
717     } else if (ifStack_.empty()) {
718       prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
719           "#endif: no #if, #ifdef, or #ifndef"_err_en_US);
720     } else {
721       ifStack_.pop();
722     }
723   } else if (dirName == "error") {
724     prescanner.Say(
725         dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
726         "%s"_err_en_US, dir.ToString());
727   } else if (dirName == "warning") {
728     prescanner.Say(
729         dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
730         "%s"_warn_en_US, dir.ToString());
731   } else if (dirName == "comment" || dirName == "note") {
732     prescanner.Say(
733         dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
734         "%s"_en_US, dir.ToString());
735   } else if (dirName == "include") {
736     if (j == tokens) {
737       prescanner.Say(
738           dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
739           "#include: missing name of file to include"_err_en_US);
740       return;
741     }
742     std::optional<std::string> prependPath;
743     TokenSequence path{dir, j, tokens - j};
744     std::string include{path.TokenAt(0).ToString()};
745     if (include != "<" && include.substr(0, 1) != "\"" &&
746         include.substr(0, 1) != "'") {
747       path = ReplaceMacros(path, prescanner);
748       include = path.empty() ? ""s : path.TokenAt(0).ToString();
749     }
750     auto pathTokens{path.SizeInTokens()};
751     std::size_t k{0};
752     if (include == "<") { // #include <foo>
753       k = 1;
754       if (k >= pathTokens) {
755         prescanner.Say(dir.GetIntervalProvenanceRange(j, pathTokens),
756             "#include: file name missing"_err_en_US);
757         return;
758       }
759       while (k < pathTokens && path.TokenAt(k) != ">") {
760         ++k;
761       }
762       if (k >= pathTokens) {
763         if (prescanner.features().ShouldWarn(
764                 common::UsageWarning::Portability)) {
765           prescanner.Say(common::UsageWarning::Portability,
766               dir.GetIntervalProvenanceRange(j, tokens - j),
767               "#include: expected '>' at end of included file"_port_en_US);
768         }
769       }
770       TokenSequence braced{path, 1, k - 1};
771       include = braced.ToString();
772     } else if ((include.substr(0, 1) == "\"" || include.substr(0, 1) == "'") &&
773         include.front() == include.back()) {
774       // #include "foo" and #include 'foo'
775       include = include.substr(1, include.size() - 2);
776       // Start search in directory of file containing the directive
777       auto prov{dir.GetTokenProvenanceRange(dirOffset).start()};
778       if (const auto *currentFile{allSources_.GetSourceFile(prov)}) {
779         prependPath = DirectoryName(currentFile->path());
780       }
781     } else {
782       prescanner.Say(dir.GetTokenProvenanceRange(j < tokens ? j : tokens - 1),
783           "#include %s: expected name of file to include"_err_en_US,
784           path.ToString());
785       return;
786     }
787     if (include.empty()) {
788       prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
789           "#include %s: empty include file name"_err_en_US, path.ToString());
790       return;
791     }
792     k = path.SkipBlanks(k + 1);
793     if (k < pathTokens && path.TokenAt(k).ToString() != "!") {
794       if (prescanner.features().ShouldWarn(common::UsageWarning::Portability)) {
795         prescanner.Say(common::UsageWarning::Portability,
796             dir.GetIntervalProvenanceRange(j, tokens - j),
797             "#include: extra stuff ignored after file name"_port_en_US);
798       }
799     }
800     std::string buf;
801     llvm::raw_string_ostream error{buf};
802     if (const SourceFile *
803         included{allSources_.Open(include, error, std::move(prependPath))}) {
804       if (included->bytes() > 0) {
805         ProvenanceRange fileRange{
806             allSources_.AddIncludedFile(*included, dir.GetProvenanceRange())};
807         Prescanner{prescanner, *this, /*isNestedInIncludeDirective=*/true}
808             .set_encoding(included->encoding())
809             .Prescan(fileRange);
810       }
811     } else {
812       prescanner.Say(dir.GetTokenProvenanceRange(j), "#include: %s"_err_en_US,
813           error.str());
814     }
815   } else {
816     prescanner.Say(dir.GetTokenProvenanceRange(dirOffset),
817         "#%s: unknown or unimplemented directive"_err_en_US, dirName);
818   }
819 }
820 
821 void Preprocessor::PrintMacros(llvm::raw_ostream &out) const {
822   // std::set is ordered. Use that to print the macros in an
823   // alphabetical order.
824   std::set<std::string> macroNames;
825   for (const auto &[name, _] : definitions_) {
826     macroNames.insert(name.ToString());
827   }
828 
829   for (const std::string &name : macroNames) {
830     out << "#define " << name;
831     definitions_.at(name).Print(out, name.c_str());
832     out << '\n';
833   }
834 }
835 
836 CharBlock Preprocessor::SaveTokenAsName(const CharBlock &t) {
837   names_.push_back(t.ToString());
838   return {names_.back().data(), names_.back().size()};
839 }
840 
841 bool Preprocessor::IsNameDefined(const CharBlock &token) {
842   return definitions_.find(token) != definitions_.end();
843 }
844 
845 bool Preprocessor::IsFunctionLikeDefinition(const CharBlock &token) {
846   auto it{definitions_.find(token)};
847   return it != definitions_.end() && it->second.isFunctionLike();
848 }
849 
850 static std::string GetDirectiveName(
851     const TokenSequence &line, std::size_t *rest) {
852   std::size_t tokens{line.SizeInTokens()};
853   std::size_t j{line.SkipBlanks(0)};
854   if (j == tokens || line.TokenAt(j).ToString() != "#") {
855     *rest = tokens;
856     return "";
857   }
858   j = line.SkipBlanks(j + 1);
859   if (j == tokens) {
860     *rest = tokens;
861     return "";
862   }
863   *rest = line.SkipBlanks(j + 1);
864   return ToLowerCaseLetters(line.TokenAt(j).ToString());
865 }
866 
867 void Preprocessor::SkipDisabledConditionalCode(const std::string &dirName,
868     IsElseActive isElseActive, Prescanner &prescanner,
869     ProvenanceRange provenanceRange) {
870   int nesting{0};
871   while (!prescanner.IsAtEnd()) {
872     if (!prescanner.IsNextLinePreprocessorDirective()) {
873       prescanner.NextLine();
874       continue;
875     }
876     TokenSequence line{prescanner.TokenizePreprocessorDirective()};
877     std::size_t rest{0};
878     std::string dn{GetDirectiveName(line, &rest)};
879     if (dn == "ifdef" || dn == "ifndef" || dn == "if") {
880       ++nesting;
881     } else if (dn == "endif") {
882       if (nesting-- == 0) {
883         return;
884       }
885     } else if (isElseActive == IsElseActive::Yes && nesting == 0) {
886       if (dn == "else") {
887         ifStack_.push(CanDeadElseAppear::No);
888         return;
889       }
890       if (dn == "elif" &&
891           IsIfPredicateTrue(
892               line, rest, line.SizeInTokens() - rest, prescanner)) {
893         ifStack_.push(CanDeadElseAppear::Yes);
894         return;
895       }
896     }
897   }
898   prescanner.Say(provenanceRange, "#%s: missing #endif"_err_en_US, dirName);
899 }
900 
901 // Precedence level codes used here to accommodate mixed Fortran and C:
902 // 15: parentheses and constants, logical !, bitwise ~
903 // 14: unary + and -
904 // 13: **
905 // 12: *, /, % (modulus)
906 // 11: + and -
907 // 10: << and >>
908 //  9: bitwise &
909 //  8: bitwise ^
910 //  7: bitwise |
911 //  6: relations (.EQ., ==, &c.)
912 //  5: .NOT.
913 //  4: .AND., &&
914 //  3: .OR., ||
915 //  2: .EQV. and .NEQV. / .XOR.
916 //  1: ? :
917 //  0: ,
918 static std::int64_t ExpressionValue(const TokenSequence &token,
919     int minimumPrecedence, std::size_t *atToken,
920     std::optional<Message> *error) {
921   enum Operator {
922     PARENS,
923     CONST,
924     NOTZERO, // !
925     COMPLEMENT, // ~
926     UPLUS,
927     UMINUS,
928     POWER,
929     TIMES,
930     DIVIDE,
931     MODULUS,
932     ADD,
933     SUBTRACT,
934     LEFTSHIFT,
935     RIGHTSHIFT,
936     BITAND,
937     BITXOR,
938     BITOR,
939     LT,
940     LE,
941     EQ,
942     NE,
943     GE,
944     GT,
945     NOT,
946     AND,
947     OR,
948     EQV,
949     NEQV,
950     SELECT,
951     COMMA
952   };
953   static const int precedence[]{
954       15, 15, 15, 15, // (), 6, !, ~
955       14, 14, // unary +, -
956       13, 12, 12, 12, 11, 11, 10, 10, // **, *, /, %, +, -, <<, >>
957       9, 8, 7, // &, ^, |
958       6, 6, 6, 6, 6, 6, // relations .LT. to .GT.
959       5, 4, 3, 2, 2, // .NOT., .AND., .OR., .EQV., .NEQV.
960       1, 0 // ?: and ,
961   };
962   static const int operandPrecedence[]{0, -1, 15, 15, 15, 15, 13, 12, 12, 12,
963       11, 11, 11, 11, 9, 8, 7, 7, 7, 7, 7, 7, 7, 6, 4, 3, 3, 3, 1, 0};
964 
965   static std::map<std::string, enum Operator> opNameMap;
966   if (opNameMap.empty()) {
967     opNameMap["("] = PARENS;
968     opNameMap["!"] = NOTZERO;
969     opNameMap["~"] = COMPLEMENT;
970     opNameMap["**"] = POWER;
971     opNameMap["*"] = TIMES;
972     opNameMap["/"] = DIVIDE;
973     opNameMap["%"] = MODULUS;
974     opNameMap["+"] = ADD;
975     opNameMap["-"] = SUBTRACT;
976     opNameMap["<<"] = LEFTSHIFT;
977     opNameMap[">>"] = RIGHTSHIFT;
978     opNameMap["&"] = BITAND;
979     opNameMap["^"] = BITXOR;
980     opNameMap["|"] = BITOR;
981     opNameMap[".lt."] = opNameMap["<"] = LT;
982     opNameMap[".le."] = opNameMap["<="] = LE;
983     opNameMap[".eq."] = opNameMap["=="] = EQ;
984     opNameMap[".ne."] = opNameMap["/="] = opNameMap["!="] = NE;
985     opNameMap[".ge."] = opNameMap[">="] = GE;
986     opNameMap[".gt."] = opNameMap[">"] = GT;
987     opNameMap[".not."] = NOT;
988     opNameMap[".and."] = opNameMap[".a."] = opNameMap["&&"] = AND;
989     opNameMap[".or."] = opNameMap[".o."] = opNameMap["||"] = OR;
990     opNameMap[".eqv."] = EQV;
991     opNameMap[".neqv."] = opNameMap[".xor."] = opNameMap[".x."] = NEQV;
992     opNameMap["?"] = SELECT;
993     opNameMap[","] = COMMA;
994   }
995 
996   std::size_t tokens{token.SizeInTokens()};
997   CHECK(tokens > 0);
998   if (*atToken >= tokens) {
999     *error =
1000         Message{token.GetProvenanceRange(), "incomplete expression"_err_en_US};
1001     return 0;
1002   }
1003 
1004   // Parse and evaluate a primary or a unary operator and its operand.
1005   std::size_t opAt{*atToken};
1006   std::string t{token.TokenAt(opAt).ToString()};
1007   enum Operator op;
1008   std::int64_t left{0};
1009   if (t == "(") {
1010     op = PARENS;
1011   } else if (IsDecimalDigit(t[0])) {
1012     op = CONST;
1013     std::size_t consumed{0};
1014     left = std::stoll(t, &consumed, 0 /*base to be detected*/);
1015     if (consumed < t.size()) {
1016       *error = Message{token.GetTokenProvenanceRange(opAt),
1017           "Uninterpretable numeric constant '%s'"_err_en_US, t};
1018       return 0;
1019     }
1020   } else if (IsLegalIdentifierStart(t[0])) {
1021     // undefined macro name -> zero
1022     // TODO: BOZ constants?
1023     op = CONST;
1024   } else if (t == "+") {
1025     op = UPLUS;
1026   } else if (t == "-") {
1027     op = UMINUS;
1028   } else if (t == "." && *atToken + 2 < tokens &&
1029       ToLowerCaseLetters(token.TokenAt(*atToken + 1).ToString()) == "not" &&
1030       token.TokenAt(*atToken + 2).ToString() == ".") {
1031     op = NOT;
1032     *atToken += 2;
1033   } else {
1034     auto it{opNameMap.find(t)};
1035     if (it != opNameMap.end()) {
1036       op = it->second;
1037     } else {
1038       *error = Message{token.GetTokenProvenanceRange(opAt),
1039           "operand expected in expression"_err_en_US};
1040       return 0;
1041     }
1042   }
1043   if (precedence[op] < minimumPrecedence) {
1044     *error = Message{token.GetTokenProvenanceRange(opAt),
1045         "operator precedence error"_err_en_US};
1046     return 0;
1047   }
1048   ++*atToken;
1049   if (op != CONST) {
1050     left = ExpressionValue(token, operandPrecedence[op], atToken, error);
1051     if (*error) {
1052       return 0;
1053     }
1054     switch (op) {
1055     case PARENS:
1056       if (*atToken < tokens && token.TokenAt(*atToken).OnlyNonBlank() == ')') {
1057         ++*atToken;
1058         break;
1059       }
1060       if (*atToken >= tokens) {
1061         *error = Message{token.GetProvenanceRange(),
1062             "')' missing from expression"_err_en_US};
1063       } else {
1064         *error = Message{
1065             token.GetTokenProvenanceRange(*atToken), "expected ')'"_err_en_US};
1066       }
1067       return 0;
1068     case NOTZERO:
1069       left = !left;
1070       break;
1071     case COMPLEMENT:
1072       left = ~left;
1073       break;
1074     case UPLUS:
1075       break;
1076     case UMINUS:
1077       left = -left;
1078       break;
1079     case NOT:
1080       left = -!left;
1081       break;
1082     default:
1083       CRASH_NO_CASE;
1084     }
1085   }
1086 
1087   // Parse and evaluate binary operators and their second operands, if present.
1088   while (*atToken < tokens) {
1089     int advance{1};
1090     t = token.TokenAt(*atToken).ToString();
1091     if (t == "." && *atToken + 2 < tokens &&
1092         token.TokenAt(*atToken + 2).ToString() == ".") {
1093       t += ToLowerCaseLetters(token.TokenAt(*atToken + 1).ToString()) + '.';
1094       advance = 3;
1095     }
1096     auto it{opNameMap.find(t)};
1097     if (it == opNameMap.end()) {
1098       break;
1099     }
1100     op = it->second;
1101     if (op < POWER || precedence[op] < minimumPrecedence) {
1102       break;
1103     }
1104     opAt = *atToken;
1105     *atToken += advance;
1106 
1107     std::int64_t right{
1108         ExpressionValue(token, operandPrecedence[op], atToken, error)};
1109     if (*error) {
1110       return 0;
1111     }
1112 
1113     switch (op) {
1114     case POWER:
1115       if (left == 0) {
1116         if (right < 0) {
1117           *error = Message{token.GetTokenProvenanceRange(opAt),
1118               "0 ** negative power"_err_en_US};
1119         }
1120       } else if (left != 1 && right != 1) {
1121         if (right <= 0) {
1122           left = !right;
1123         } else {
1124           std::int64_t power{1};
1125           for (; right > 0; --right) {
1126             if ((power * left) / left != power) {
1127               *error = Message{token.GetTokenProvenanceRange(opAt),
1128                   "overflow in exponentation"_err_en_US};
1129               left = 1;
1130             }
1131             power *= left;
1132           }
1133           left = power;
1134         }
1135       }
1136       break;
1137     case TIMES:
1138       if (left != 0 && right != 0 && ((left * right) / left) != right) {
1139         *error = Message{token.GetTokenProvenanceRange(opAt),
1140             "overflow in multiplication"_err_en_US};
1141       }
1142       left = left * right;
1143       break;
1144     case DIVIDE:
1145       if (right == 0) {
1146         *error = Message{
1147             token.GetTokenProvenanceRange(opAt), "division by zero"_err_en_US};
1148         left = 0;
1149       } else {
1150         left = left / right;
1151       }
1152       break;
1153     case MODULUS:
1154       if (right == 0) {
1155         *error = Message{
1156             token.GetTokenProvenanceRange(opAt), "modulus by zero"_err_en_US};
1157         left = 0;
1158       } else {
1159         left = left % right;
1160       }
1161       break;
1162     case ADD:
1163       if ((left < 0) == (right < 0) && (left < 0) != (left + right < 0)) {
1164         *error = Message{token.GetTokenProvenanceRange(opAt),
1165             "overflow in addition"_err_en_US};
1166       }
1167       left = left + right;
1168       break;
1169     case SUBTRACT:
1170       if ((left < 0) != (right < 0) && (left < 0) == (left - right < 0)) {
1171         *error = Message{token.GetTokenProvenanceRange(opAt),
1172             "overflow in subtraction"_err_en_US};
1173       }
1174       left = left - right;
1175       break;
1176     case LEFTSHIFT:
1177       if (right < 0 || right > 64) {
1178         *error = Message{token.GetTokenProvenanceRange(opAt),
1179             "bad left shift count"_err_en_US};
1180       }
1181       left = right >= 64 ? 0 : left << right;
1182       break;
1183     case RIGHTSHIFT:
1184       if (right < 0 || right > 64) {
1185         *error = Message{token.GetTokenProvenanceRange(opAt),
1186             "bad right shift count"_err_en_US};
1187       }
1188       left = right >= 64 ? 0 : left >> right;
1189       break;
1190     case BITAND:
1191     case AND:
1192       left = left & right;
1193       break;
1194     case BITXOR:
1195       left = left ^ right;
1196       break;
1197     case BITOR:
1198     case OR:
1199       left = left | right;
1200       break;
1201     case LT:
1202       left = -(left < right);
1203       break;
1204     case LE:
1205       left = -(left <= right);
1206       break;
1207     case EQ:
1208       left = -(left == right);
1209       break;
1210     case NE:
1211       left = -(left != right);
1212       break;
1213     case GE:
1214       left = -(left >= right);
1215       break;
1216     case GT:
1217       left = -(left > right);
1218       break;
1219     case EQV:
1220       left = -(!left == !right);
1221       break;
1222     case NEQV:
1223       left = -(!left != !right);
1224       break;
1225     case SELECT:
1226       if (*atToken >= tokens || token.TokenAt(*atToken).ToString() != ":") {
1227         *error = Message{token.GetTokenProvenanceRange(opAt),
1228             "':' required in selection expression"_err_en_US};
1229         return 0;
1230       } else {
1231         ++*atToken;
1232         std::int64_t third{
1233             ExpressionValue(token, operandPrecedence[op], atToken, error)};
1234         left = left != 0 ? right : third;
1235       }
1236       break;
1237     case COMMA:
1238       left = right;
1239       break;
1240     default:
1241       CRASH_NO_CASE;
1242     }
1243   }
1244   return left;
1245 }
1246 
1247 bool Preprocessor::IsIfPredicateTrue(const TokenSequence &directive,
1248     std::size_t first, std::size_t exprTokens, Prescanner &prescanner) {
1249   TokenSequence expr{directive, first, exprTokens};
1250   TokenSequence replaced{
1251       ReplaceMacros(expr, prescanner, nullptr, /*inIfExpression=*/true)};
1252   if (replaced.HasBlanks()) {
1253     replaced.RemoveBlanks();
1254   }
1255   if (replaced.empty()) {
1256     prescanner.Say(expr.GetProvenanceRange(), "empty expression"_err_en_US);
1257     return false;
1258   }
1259   std::size_t atToken{0};
1260   std::optional<Message> error;
1261   bool result{ExpressionValue(replaced, 0, &atToken, &error) != 0};
1262   if (error) {
1263     prescanner.Say(std::move(*error));
1264   } else if (atToken < replaced.SizeInTokens() &&
1265       replaced.TokenAt(atToken).ToString() != "!") {
1266     prescanner.Say(replaced.GetIntervalProvenanceRange(
1267                        atToken, replaced.SizeInTokens() - atToken),
1268         atToken == 0 ? "could not parse any expression"_err_en_US
1269                      : "excess characters after expression"_err_en_US);
1270   }
1271   return result;
1272 }
1273 
1274 void Preprocessor::LineDirective(
1275     const TokenSequence &dir, std::size_t j, Prescanner &prescanner) {
1276   std::size_t tokens{dir.SizeInTokens()};
1277   const std::string *linePath{nullptr};
1278   std::optional<int> lineNumber;
1279   SourceFile *sourceFile{nullptr};
1280   std::optional<SourcePosition> pos;
1281   for (; j < tokens; j = dir.SkipBlanks(j + 1)) {
1282     std::string tstr{dir.TokenAt(j).ToString()};
1283     Provenance provenance{dir.GetTokenProvenance(j)};
1284     if (!pos) {
1285       pos = allSources_.GetSourcePosition(provenance);
1286     }
1287     if (!sourceFile && pos) {
1288       sourceFile = const_cast<SourceFile *>(&*pos->sourceFile);
1289     }
1290     if (tstr.front() == '"' && tstr.back() == '"') {
1291       tstr = tstr.substr(1, tstr.size() - 2);
1292       if (!tstr.empty() && sourceFile) {
1293         linePath = &sourceFile->SavePath(std::move(tstr));
1294       }
1295     } else if (IsDecimalDigit(tstr[0])) {
1296       if (!lineNumber) { // ignore later column number
1297         int ln{0};
1298         for (char c : tstr) {
1299           if (IsDecimalDigit(c)) {
1300             int nln{10 * ln + c - '0'};
1301             if (nln / 10 == ln && nln % 10 == c - '0') {
1302               ln = nln;
1303               continue;
1304             }
1305           }
1306           prescanner.Say(provenance,
1307               "bad line number '%s' in #line directive"_err_en_US, tstr);
1308           return;
1309         }
1310         lineNumber = ln;
1311       }
1312     } else {
1313       prescanner.Say(
1314           provenance, "bad token '%s' in #line directive"_err_en_US, tstr);
1315       return;
1316     }
1317   }
1318   if (lineNumber && sourceFile) {
1319     CHECK(pos);
1320     if (!linePath) {
1321       linePath = &*pos->path;
1322     }
1323     sourceFile->LineDirective(pos->trueLineNumber + 1, *linePath, *lineNumber);
1324   }
1325 }
1326 
1327 } // namespace Fortran::parser
1328