[flang] Honor #line and related preprocessing directives

[klausler]

Summary

Extend the SourceFile class to take account of #line directives
when computing source file positions for error messages.
Adjust the output of #line directives to -E output so that they
reflect any #line directives that were in the input.

[tschuett]

flang/include/flang/Parser/source.h

89

llvm::StringSet?

flang/lib/Parser/provenance.cpp

533

llvm::SmallPtrSet?

[klausler]

Update FlangOmpReportVisitor.cpp.

[domada]

LGTM.
Thank you for your patch.
I tested it, and it solves all issues connected with -save-temps flag, which were mentioned on LLVM discourse.
Could you fix one check statement in flang/test/Parser/line-directive.f90 which fails on Windows?

flang/include/flang/Parser/source.h

75

nit: llvm::StringRef?

flang/test/Parser/line-directive.f90

3

This check fails on Windows.

Input was:
<<<<<<
           1: #line "C:\ws\w3\llvm-project\premerge-checks\flang\test\Parser\line-directive.f90" 3
check:2'0     X~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ error: no match found
check:2'1                                                             ?                             possible intended match
           2:  subroutine s
check:2'0     ~~~~~~~~~~~~~~
           3:  implicit none
check:2'0     ~~~~~~~~~~~~~~~
           4:  a = 1.
check:2'0     ~~~~~~~~
           5: #line 101
check:2'0     ~~~~~~~~~~
           6:  b = 2.
check:2'0     ~~~~~~~~
           .
           .
           .
>>>>>>

Source: https://buildkite.com/llvm-project/premerge-checks/builds/160946#0188fefb-5134-482b-bece-1e8c7632b903 log line: 6653

[kiranchandramohan]

flang/lib/Parser/preprocessor.cpp

1131

Nit: thisTrueLineNumber is unused.

[klausler]

flang/include/flang/Parser/source.h

75

I use only standard C++ library features in parts of the compiler that people may want to use to build tooling outside the LLVM ecosystem, when those features are universally portable.

flang/lib/Parser/preprocessor.cpp

1131

It is now indeed superfluous; thanks for noticing.

flang/test/Parser/line-directive.f90

3

DOS backslashes strike again. Will fix.

[klausler]

Address review comments.

[tschuett]

Why? Nobody stops you from building flang-tools-extras with LSP-servers, refactoring, Flang-tidy you name it. Flang is in the LLVM monorepo. Why do you refuse to use LLVM data structures?

[klausler]

In D153910#4455483, @domada wrote:

LGTM.
Thank you for your patch.
I tested it, and it solves all issues connected with -save-temps flag, which were mentioned on LLVM discourse.

This kind of testing is the most useful kind of code review; thanks for confirming that your problem will be solved. I've updated the patch and it should now clear the pre-merge testing.

[klausler]

Fix FileCheck regular expression for real this time; sorry for the distraction.

[klausler]

The pre-merge testing bug has been resolved. Please approve when convenient; thanks!

[kiranchandramohan]

LG. Thanks, this helps unblock engineers working on OpenMP target offloading.

[domada]

LGTM

flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp

void OpenMPCounterVisitor::Post(const OpenMPConstruct &) {
   PostConstructsCommon();
 }
 void OpenMPCounterVisitor::PostConstructsCommon() {
   OmpWrapperType *curConstruct = ompWrapperStack.back();
   std::sort(
       clauseStrings[curConstruct].begin(), clauseStrings[curConstruct].end());
 
   SourcePosition s{getLocation(*curConstruct)};
-  LogRecord r{s.file.path(), s.line, getName(*curConstruct),
-      clauseStrings[curConstruct]};
+  LogRecord r{
+      s.path, s.line, getName(*curConstruct), clauseStrings[curConstruct]};
   constructClauses.push_back(r);
 
   auto it = clauseStrings.find(curConstruct);
   clauseStrings.erase(it);
   ompWrapperStack.pop_back();
   delete curConstruct;
 }
 

flang/include/flang/Parser/source.h

 #define FORTRAN_PARSER_SOURCE_H_
 
 // Source file content is lightly normalized when the file is read.
 //  - Line ending markers are converted to single newline characters
 //  - A newline character is added to the last line of the file if one is needed
 //  - A Unicode byte order mark is recognized if present.
 
 #include "characters.h"
+#include "flang/Common/reference.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
 #include <cstddef>
 #include <list>
+#include <map>
 #include <optional>
+#include <set>
 #include <string>
 #include <utility>
 #include <vector>
 
 namespace llvm {
 class raw_ostream;
 }
 
 namespace Fortran::parser {
 
 std::string DirectoryName(std::string path);
 std::optional<std::string> LocateSourceFile(
     std::string name, const std::list<std::string> &searchPath);
 
 class SourceFile;
 
 struct SourcePosition {
-  const SourceFile &file;
+  common::Reference<const SourceFile> sourceFile;
+  common::Reference<const std::string>
+      path; // may not be sourceFile.path() when #line present
   int line, column;
+  int trueLineNumber;
 };
 
 class SourceFile {
 public:
   explicit SourceFile(Encoding e) : encoding_{e} {}
   ~SourceFile();
-  std::string path() const { return path_; }
+  const std::string &path() const { return path_; }
   llvm::ArrayRef<char> content() const {
     return buf_->getBuffer().slice(bom_end_, buf_end_ - bom_end_);
   }
   std::size_t bytes() const { return content().size(); }
   std::size_t lines() const { return lineStart_.size(); }
   Encoding encoding() const { return encoding_; }
 
   bool Open(std::string path, llvm::raw_ostream &error);
   bool ReadStandardInput(llvm::raw_ostream &error);
   void Close();
-  SourcePosition FindOffsetLineAndColumn(std::size_t) const;
+  SourcePosition GetSourcePosition(std::size_t) const;
   std::size_t GetLineStartOffset(int lineNumber) const {
     return lineStart_.at(lineNumber - 1);
   }
+  const std::string &SavePath(std::string &&);
+  void LineDirective(int trueLineNumber, const std::string &, int);
+  llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
 
 private:
+  struct SourcePositionOrigin {
+    const std::string &path;
+    int line;
+  };
+
   void ReadFile();
   void IdentifyPayload();
   void RecordLineStarts();
 
   std::string path_;
   std::unique_ptr<llvm::WritableMemoryBuffer> buf_;
   std::vector<std::size_t> lineStart_;
   std::size_t bom_end_{0};
   std::size_t buf_end_;
   Encoding encoding_;
+  std::set<std::string> distinctPaths_;
+  std::map<std::size_t, SourcePositionOrigin> origins_;
 };
 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_SOURCE_H_

flang/lib/Frontend/FrontendActions.cpp

void GetDefinitionAction::executeAction() {
   if (!sourceInfo) {
     llvm_unreachable(
         "Failed to obtain SourcePosition."
         "TODO: Please, write a test and replace this with a diagnostic!");
     return;
   }
 
   llvm::outs() << "Found symbol name: " << symbol->name().ToString() << "\n";
-  llvm::outs() << symbol->name().ToString() << ": "
-               << sourceInfo->first.file.path() << ", "
-               << sourceInfo->first.line << ", " << sourceInfo->first.column
-               << "-" << sourceInfo->second.column << "\n";
+  llvm::outs() << symbol->name().ToString() << ": " << sourceInfo->first.path
+               << ", " << sourceInfo->first.line << ", "
+               << sourceInfo->first.column << "-" << sourceInfo->second.column
+               << "\n";
 }
 
 void GetSymbolsSourcesAction::executeAction() {
   CompilerInstance &ci = this->getInstance();
 
   // Report and exit if fatal semantic errors are present
   if (reportFatalSemanticErrors()) {
     return;

flang/lib/Lower/Bridge.cpp

public:
   genLocation(const Fortran::parser::CharBlock &block) override final {
     if (const Fortran::parser::AllCookedSources *cooked =
             bridge.getCookedSource()) {
       if (std::optional<std::pair<Fortran::parser::SourcePosition,
                                   Fortran::parser::SourcePosition>>
               loc = cooked->GetSourcePositionRange(block)) {
         // loc is a pair (begin, end); use the beginning position
         Fortran::parser::SourcePosition &filePos = loc->first;
-        llvm::SmallString<256> filePath(filePos.file.path());
+        llvm::SmallString<256> filePath(*filePos.path);
         llvm::sys::fs::make_absolute(filePath);
         llvm::sys::path::remove_dots(filePath);
         return mlir::FileLineColLoc::get(&getMLIRContext(), filePath.str(),
                                          filePos.line, filePos.column);
       }
     }
     return genUnknownLocation();
   }

flang/lib/Parser/parsing.cpp

const SourceFile *Parsing::Prescan(const std::string &path, Options options) {
   if (options.isModuleFile) {
     for (const auto &path : options.searchDirectories) {
       allSources.AppendSearchPathDirectory(path);
     }
   }
 
   std::string buf;
   llvm::raw_string_ostream fileError{buf};
-  const SourceFile *sourceFile;
+  const SourceFile *sourceFile{nullptr};
   if (path == "-") {
     sourceFile = allSources.ReadStandardInput(fileError);
   } else if (options.isModuleFile) {
     // Don't mess with intrinsic module search path
     sourceFile = allSources.Open(path, fileError);
   } else {
     sourceFile =
         allSources.Open(path, fileError, "."s /*prepend to search path*/);
const SourceFile *Parsing::Prescan(const std::string &path, Options options) {
   if (options.showColors) {
     allSources.setShowColors(/*showColors=*/true);
   }
   return sourceFile;
 }
 
 void Parsing::EmitPreprocessedSource(
     llvm::raw_ostream &out, bool lineDirectives) const {
-  const SourceFile *sourceFile{nullptr};
+  const std::string *sourcePath{nullptr};
   int sourceLine{0};
   int column{1};
   bool inDirective{false};
   bool inContinuation{false};
   bool lineWasBlankBefore{true};
   const AllSources &allSources{allCooked().allSources()};
   // All directives that flang support are known to have a length of 3 chars
   constexpr int directiveNameLength{3};
if (ch == '\n') {
           IsLetter(ch)) {
         directive += getOriginalChar(ch);
       }
 
       std::optional<SourcePosition> position{provenance
               ? allSources.GetSourcePosition(provenance->start())
               : std::nullopt};
       if (lineDirectives && column == 1 && position) {
-        if (&position->file != sourceFile) {
-          out << "#line \"" << position->file.path() << "\" " << position->line
+        if (&*position->path != sourcePath) {
+          out << "#line \"" << *position->path << "\" " << position->line
               << '\n';
         } else if (position->line != sourceLine) {
           if (sourceLine < position->line &&
               sourceLine + 10 >= position->line) {
             // Emit a few newlines to catch up when they'll likely
             // require fewer bytes than a #line directive would have
             // occupied.
             while (sourceLine++ < position->line) {
               out << '\n';
             }
           } else {
             out << "#line " << position->line << '\n';
           }
         }
-        sourceFile = &position->file;
+        sourcePath = &*position->path;
         sourceLine = position->line;
       }
       if (column > 72) {
         // Wrap long lines in a portable fashion that works in both
         // of the Fortran source forms. The first free-form continuation
         // marker ("&") lands in column 73, which begins the card commentary
         // field of fixed form, and the second one is put in column 6,
         // where it signifies fixed form line continuation.

flang/lib/Parser/preprocessor.h

private:
   enum class CanDeadElseAppear { No, Yes };
 
   CharBlock SaveTokenAsName(const CharBlock &);
   TokenSequence ReplaceMacros(const TokenSequence &, Prescanner &);
   void SkipDisabledConditionalCode(
       const std::string &, IsElseActive, Prescanner &, ProvenanceRange);
   bool IsIfPredicateTrue(const TokenSequence &expr, std::size_t first,
       std::size_t exprTokens, Prescanner &);
+  void LineDirective(const TokenSequence &, std::size_t, Prescanner &);
 
   AllSources &allSources_;
   std::list<std::string> names_;
   std::unordered_map<CharBlock, Definition> definitions_;
   std::stack<CanDeadElseAppear> ifStack_;
 };
 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_PREPROCESSOR_H_

flang/lib/Parser/preprocessor.cpp

void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
   j = dir.SkipBlanks(j + 1);
   while (tokens > 0 && dir.TokenAt(tokens - 1).IsBlank()) {
     --tokens;
   }
   if (j == tokens) {
     return;
   }
   if (IsDecimalDigit(dir.TokenAt(j)[0]) || dir.TokenAt(j)[0] == '"') {
-    return; // treat like #line, ignore it
+    LineDirective(dir, j, prescanner);
+    return;
   }
   std::size_t dirOffset{j};
   std::string dirName{ToLowerCaseLetters(dir.TokenAt(dirOffset).ToString())};
   j = dir.SkipBlanks(j + 1);
   CharBlock nameToken;
   if (j < tokens && IsLegalIdentifierStart(dir.TokenAt(j)[0])) {
     nameToken = dir.TokenAt(j);
   }
   if (dirName == "line") {
-    // #line is ignored
+    LineDirective(dir, j, prescanner);
   } else if (dirName == "define") {
     if (nameToken.empty()) {
       prescanner.Say(dir.GetTokenProvenanceRange(j < tokens ? j : tokens - 1),
           "#define: missing or invalid name"_err_en_US);
       return;
     }
     nameToken = SaveTokenAsName(nameToken);
     definitions_.erase(nameToken);
prescanner.Say(std::move(*error));
       expr3.TokenAt(atToken).ToString() != "!") {
     prescanner.Say(expr3.GetIntervalProvenanceRange(
                        atToken, expr3.SizeInTokens() - atToken),
         atToken == 0 ? "could not parse any expression"_err_en_US
                      : "excess characters after expression"_err_en_US);
   }
   return result;
 }
+
+void Preprocessor::LineDirective(
+    const TokenSequence &dir, std::size_t j, Prescanner &prescanner) {
+  std::size_t tokens{dir.SizeInTokens()};
+  const std::string *linePath{nullptr};
+  std::optional<int> lineNumber;
+  SourceFile *sourceFile{nullptr};
+  std::optional<SourcePosition> pos;
+  for (; j < tokens; j = dir.SkipBlanks(j + 1)) {
+    std::string tstr{dir.TokenAt(j).ToString()};
+    Provenance provenance{dir.GetTokenProvenance(j)};
+    if (!pos) {
+      pos = allSources_.GetSourcePosition(provenance);
+    }
+    if (!sourceFile && pos) {
+      sourceFile = const_cast<SourceFile *>(&*pos->sourceFile);
+    }
+    if (tstr.front() == '"' && tstr.back() == '"') {
+      tstr = tstr.substr(1, tstr.size() - 2);
+      if (!tstr.empty() && sourceFile) {
+        linePath = &sourceFile->SavePath(std::move(tstr));
+      }
+    } else if (IsDecimalDigit(tstr[0])) {
+      if (!lineNumber) { // ignore later column number
+        int ln{0};
+        for (char c : tstr) {
+          if (IsDecimalDigit(c)) {
+            int nln{10 * ln + c - '0'};
+            if (nln / 10 == ln && nln % 10 == c - '0') {
+              ln = nln;
+              continue;
+            }
+          }
+          prescanner.Say(provenance,
+              "bad line number '%s' in #line directive"_err_en_US, tstr);
+          return;
+        }
+        lineNumber = ln;
+      }
+    } else {
+      prescanner.Say(
+          provenance, "bad token '%s' in #line directive"_err_en_US, tstr);
+      return;
+    }
+  }
+  if (lineNumber && sourceFile) {
+    CHECK(pos);
+    if (!linePath) {
+      linePath = &*pos->path;
+    }
+    sourceFile->LineDirective(pos->trueLineNumber + 1, *linePath, *lineNumber);
+  }
+}
 } // namespace Fortran::parser

flang/lib/Parser/prescan.h

private:
   int continuationLines_{0};
 
   Provenance startProvenance_;
   const char *start_{nullptr}; // beginning of current source file content
   const char *limit_{nullptr}; // first address after end of current source
   const char *nextLine_{nullptr}; // next line to process; <= limit_
   const char *directiveSentinel_{nullptr}; // current compiler directive
 
-  // This data members are state for processing the source line containing
+  // These data members are state for processing the source line containing
   // "at_", which goes to up to the newline character before "nextLine_".
   const char *at_{nullptr}; // next character to process; < nextLine_
   int column_{1}; // card image column position of next character
   bool tabInCurrentLine_{false};
   bool slashInCurrentStatement_{false};
   bool preventHollerith_{false}; // CHARACTER*4HIMOM not Hollerith
   bool inCharLiteral_{false};
   bool inPreprocessorDirective_{false};

flang/lib/Parser/prescan.cpp

if (j + 3 > n || std::memcmp(str + j, "end", 3) != 0) {
     return;
   }
   // It starts with END, possibly after a label.
   auto start{allSources_.GetSourcePosition(tokens.GetCharProvenance(j))};
   auto end{allSources_.GetSourcePosition(tokens.GetCharProvenance(n - 1))};
   if (!start || !end) {
     return;
   }
-  if (&start->file == &end->file && start->line == end->line) {
+  if (&*start->sourceFile == &*end->sourceFile && start->line == end->line) {
     return; // no continuation
   }
   j += 3;
   static const char *const prefixes[]{"program", "subroutine", "function",
       "blockdata", "module", "submodule", nullptr};
   bool isPrefix{j == n || !IsLegalInIdentifier(str[j])}; // prefix is END
   std::size_t endOfPrefix{j - 1};
   for (const char *const *p{prefixes}; *p; ++p) {
if (isPrefix) {
     auto range{tokens.GetTokenProvenanceRange(1)};
     if (j == n) { // END or END thing [name]
       Say(range,
           "Program unit END statement may not be continued in fixed form source"_err_en_US);
     } else {
       auto endOfPrefixPos{
           allSources_.GetSourcePosition(tokens.GetCharProvenance(endOfPrefix))};
       auto next{allSources_.GetSourcePosition(tokens.GetCharProvenance(j))};
-      if (endOfPrefixPos && next && &endOfPrefixPos->file == &start->file &&
+      if (endOfPrefixPos && next &&
+          &*endOfPrefixPos->sourceFile == &*start->sourceFile &&
           endOfPrefixPos->line == start->line &&
-          (&next->file != &start->file || next->line != start->line)) {
+          (&*next->sourceFile != &*start->sourceFile ||
+              next->line != start->line)) {
         Say(range,
             "Initial line of continued statement must not appear to be a program unit END in fixed form source"_err_en_US);
       }
     }
   }
 }
 
 void Prescanner::SkipToEndOfLine() {

flang/lib/Parser/provenance.cpp

 //===-- lib/Parser/provenance.cpp -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "flang/Parser/provenance.h"
 #include "flang/Common/idioms.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
+#include <set>
 #include <utility>
 
 namespace Fortran::parser {
 
 ProvenanceRangeToOffsetMappings::ProvenanceRangeToOffsetMappings() {}
 ProvenanceRangeToOffsetMappings::~ProvenanceRangeToOffsetMappings() {}
 
 void ProvenanceRangeToOffsetMappings::Put(
if (!range) {
     o << message << '\n';
     return;
   }
   CHECK(IsValid(*range));
   const Origin &origin{MapToOrigin(range->start())};
   common::visit(
       common::visitors{
           [&](const Inclusion &inc) {
-            o << inc.source.path();
             std::size_t offset{origin.covers.MemberOffset(range->start())};
-            SourcePosition pos{inc.source.FindOffsetLineAndColumn(offset)};
-            o << ':' << pos.line << ':' << pos.column << ": ";
+            SourcePosition pos{inc.source.GetSourcePosition(offset)};
+            o << pos.path << ':' << pos.line << ':' << pos.column << ": ";
             EmitPrefix(o, color, prefix, this->getShowColors());
             o << message << '\n';
             if (echoSourceLine) {
               const char *text{inc.source.content().data() +
-                  inc.source.GetLineStartOffset(pos.line)};
+                  inc.source.GetLineStartOffset(pos.trueLineNumber)};
               o << "  ";
               for (const char *p{text}; *p != '\n'; ++p) {
                 o << *p;
               }
               o << "\n  ";
               for (int j{1}; j < pos.column; ++j) {
                 char ch{text[j - 1]};
                 o << (ch == '\t' ? '\t' : ' ');
               }
               o << '^';
               if (range->size() > 1) {
                 auto last{range->start() + range->size() - 1};
                 if (&MapToOrigin(last) == &origin) {
                   auto endOffset{origin.covers.MemberOffset(last)};
-                  auto endPos{inc.source.FindOffsetLineAndColumn(endOffset)};
+                  auto endPos{inc.source.GetSourcePosition(endOffset)};
                   if (pos.line == endPos.line) {
                     for (int j{pos.column}; j < endPos.column; ++j) {
                       o << '^';
                     }
                   }
                 }
               }
               o << '\n';

 
 std::optional<SourcePosition> AllSources::GetSourcePosition(
     Provenance prov) const {
   const Origin &origin{MapToOrigin(prov)};
   return common::visit(
       common::visitors{
           [&](const Inclusion &inc) -> std::optional<SourcePosition> {
             std::size_t offset{origin.covers.MemberOffset(prov)};
-            return inc.source.FindOffsetLineAndColumn(offset);
+            return inc.source.GetSourcePosition(offset);
           },
           [&](const Macro &) {
             return GetSourcePosition(origin.replaces.start());
           },
           [](const CompilerInsertion &) -> std::optional<SourcePosition> {
             return std::nullopt;
           },
       },
       origin.u);
 }
 
 std::optional<ProvenanceRange> AllSources::GetFirstFileProvenance() const {
   for (const auto &origin : origin_) {
     if (std::holds_alternative<Inclusion>(origin.u)) {
       return origin.covers;
     }
   }
   return std::nullopt;
 }
 
 std::string AllSources::GetPath(Provenance at) const {
-  const SourceFile *source{GetSourceFile(at)};
-  return source ? source->path() : ""s;
+  std::size_t offset{0};
+  const SourceFile *source{GetSourceFile(at, &offset)};
+  return source ? *source->GetSourcePosition(offset).path : ""s;
 }
 
 int AllSources::GetLineNumber(Provenance at) const {
   std::size_t offset{0};
   const SourceFile *source{GetSourceFile(at, &offset)};
-  return source ? source->FindOffsetLineAndColumn(offset).line : 0;
+  return source ? source->GetSourcePosition(offset).line : 0;
 }
 
 Provenance AllSources::CompilerInsertionProvenance(char ch) {
   auto iter{compilerInsertionProvenance_.find(ch)};
   if (iter != compilerInsertionProvenance_.end()) {
     return iter->second;
   }
   ProvenanceRange newCharRange{AddCompilerInsertion(std::string{ch})};
llvm::raw_ostream &OffsetToProvenanceMappings::Dump(
   }
   return o;
 }
 
 llvm::raw_ostream &AllSources::Dump(llvm::raw_ostream &o) const {
   o << "AllSources range_ ";
   DumpRange(o, range_);
   o << '\n';
+  std::set<const SourceFile *> sources;
   for (const Origin &m : origin_) {
     o << "   ";
     DumpRange(o, m.covers);
     o << " -> ";
     common::visit(common::visitors{
                       [&](const Inclusion &inc) {
                         if (inc.isModule) {
                           o << "module ";
                         }
                         o << "file " << inc.source.path();
+                        sources.emplace(&inc.source);
                       },
                       [&](const Macro &mac) { o << "macro " << mac.expansion; },
                       [&](const CompilerInsertion &ins) {
                         o << "compiler '" << ins.text << '\'';
                         if (ins.text.length() == 1) {
                           int ch = ins.text[0];
                           o << "(0x";
                           o.write_hex(ch & 0xff) << ")";
                         }
                       },
                   },
         m.u);
     if (IsValid(m.replaces)) {
       o << " replaces ";
       DumpRange(o, m.replaces);
     }
     o << '\n';
   }
+  for (const SourceFile *sf : sources) {
+    sf->Dump(o);
+  }
   return o;
 }
 
 llvm::raw_ostream &CookedSource::Dump(llvm::raw_ostream &o) const {
   o << "CookedSource::provenanceMap_:\n";
   provenanceMap_.Dump(o);
   o << "CookedSource::invertedMap_:\n";
   invertedMap_.Dump(o);

flang/lib/Parser/source.cpp

 //===-- lib/Parser/source.cpp ---------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "flang/Parser/source.h"
 #include "flang/Common/idioms.h"
 #include "flang/Parser/char-buffer.h"
+#include "flang/Parser/characters.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
+#include <cstring>
 #include <memory>
+#include <string>
 #include <vector>
 
 namespace Fortran::parser {
 
 SourceFile::~SourceFile() { Close(); }
 
-static std::vector<std::size_t> FindLineStarts(llvm::StringRef source) {
-  std::vector<std::size_t> result;
-  if (source.size() > 0) {
-    CHECK(source.back() == '\n' && "missing ultimate newline");
+void SourceFile::RecordLineStarts() {
+  if (std::size_t chars{bytes()}; chars > 0) {
+    origins_.emplace(1, SourcePositionOrigin{path_, 1});
+    const char *source{content().data()};
+    CHECK(source[chars - 1] == '\n' && "missing ultimate newline");
     std::size_t at{0};
-    do {
-      result.push_back(at);
-      at = source.find('\n', at) + 1;
-    } while (at < source.size());
-    result.shrink_to_fit();
-  }
-  return result;
+    do { // "at" is always at the beginning of a source line
+      lineStart_.push_back(at);
+      at = reinterpret_cast<const char *>(
+               std::memchr(source + at, '\n', chars - at)) -
+          source + 1;
+    } while (at < chars);
+    CHECK(at == chars);
+    lineStart_.shrink_to_fit();
   }
-
-void SourceFile::RecordLineStarts() {
-  lineStart_ = FindLineStarts({content().data(), bytes()});
 }
 
 // Check for a Unicode byte order mark (BOM).
 // Module files all have one; so can source files.
 void SourceFile::IdentifyPayload() {
   llvm::StringRef content{buf_->getBufferStart(), buf_->getBufferSize()};
   constexpr llvm::StringLiteral UTF8_BOM{"\xef\xbb\xbf"};
   if (content.startswith(UTF8_BOM)) {
void SourceFile::ReadFile() {
   }
   IdentifyPayload();
   RecordLineStarts();
 }
 
 void SourceFile::Close() {
   path_.clear();
   buf_.reset();
+  distinctPaths_.clear();
+  origins_.clear();
 }
 
-SourcePosition SourceFile::FindOffsetLineAndColumn(std::size_t at) const {
+SourcePosition SourceFile::GetSourcePosition(std::size_t at) const {
   CHECK(at < bytes());
+  auto it{llvm::upper_bound(lineStart_, at)};
+  auto trueLineNumber{std::distance(lineStart_.begin(), it - 1) + 1};
+  auto ub{origins_.upper_bound(trueLineNumber)};
+  auto column{static_cast<int>(at - lineStart_[trueLineNumber - 1] + 1)};
+  if (ub == origins_.begin()) {
+    return {*this, path_, static_cast<int>(trueLineNumber), column,
+        static_cast<int>(trueLineNumber)};
+  } else {
+    --ub;
+    const SourcePositionOrigin &origin{ub->second};
+    auto lineNumber{
+        trueLineNumber - ub->first + static_cast<std::size_t>(origin.line)};
+    return {*this, origin.path, static_cast<int>(lineNumber), column,
+        static_cast<int>(trueLineNumber)};
+  }
+}
+
+const std::string &SourceFile::SavePath(std::string &&path) {
+  return *distinctPaths_.emplace(std::move(path)).first;
+}
 
-  auto it = llvm::upper_bound(lineStart_, at);
-  auto low = std::distance(lineStart_.begin(), it - 1);
-  return {*this, static_cast<int>(low + 1),
-      static_cast<int>(at - lineStart_[low] + 1)};
+void SourceFile::LineDirective(
+    int trueLineNumber, const std::string &path, int lineNumber) {
+  origins_.emplace(trueLineNumber, SourcePositionOrigin{path, lineNumber});
+}
+
+llvm::raw_ostream &SourceFile::Dump(llvm::raw_ostream &o) const {
+  o << "SourceFile '" << path_ << "'\n";
+  for (const auto &[at, spo] : origins_) {
+    o << "  origin_[" << at << "] -> '" << spo.path << "' " << spo.line << '\n';
+  }
+  return o;
 }
 } // namespace Fortran::parser

flang/lib/Semantics/semantics.cpp

 
 void Semantics::DumpSymbolsSources(llvm::raw_ostream &os) const {
   NameToSymbolMap symbols;
   GetSymbolNames(context_.globalScope(), symbols);
   const parser::AllCookedSources &allCooked{context_.allCookedSources()};
   for (const auto &pair : symbols) {
     const Symbol &symbol{pair.second};
     if (auto sourceInfo{allCooked.GetSourcePositionRange(symbol.name())}) {
-      os << symbol.name().ToString() << ": " << sourceInfo->first.file.path()
-         << ", " << sourceInfo->first.line << ", " << sourceInfo->first.column
-         << "-" << sourceInfo->second.column << "\n";
+      os << symbol.name().ToString() << ": " << sourceInfo->first.path << ", "
+         << sourceInfo->first.line << ", " << sourceInfo->first.column << "-"
+         << sourceInfo->second.column << "\n";
     } else if (symbol.has<semantics::UseDetails>()) {
       os << symbol.name().ToString() << ": "
          << symbol.GetUltimate().owner().symbol()->name().ToString() << "\n";
     }
   }
 }
 
 void DoDumpSymbols(llvm::raw_ostream &os, const Scope &scope, int indent) {

flang/test/Driver/debug-provenance.f90

 ! Ensure argument -fdebug-dump-provenance works as expected.
 
 ! RUN: %flang_fc1 -fdebug-dump-provenance %s  2>&1 | FileCheck %s
 
 ! CHECK: AllSources:
 ! CHECK-NEXT: AllSources range_ [{{[0-9]*}}..{{[0-9]*}}] ({{[0-9]*}} bytes)
 ! CHECK-NEXT:    [1..1] (1 bytes) -> compiler '?'(0x3f)
 ! CHECK-NEXT:    [2..2] (1 bytes) -> compiler ' '(0x20)
 ! CHECK-NEXT:    [3..3] (1 bytes) -> compiler '\'(0x5c)
 ! CHECK-NEXT:    [{{[0-9]*}}..{{[0-9]*}}] ({{[0-9]*}} bytes) -> file {{.*[/\\]}}debug-provenance.f90
 ! CHECK-NEXT:    [{{[0-9]*}}..{{[0-9]*}}] ({{[0-9]*}} bytes) -> compiler '(after end of source)'
+! CHECK-NEXT: SourceFile '{{.*[/\\]}}debug-provenance.f90'
+! CHECK-NEXT:   origin_[1] -> '{{.*[/\\]}}debug-provenance.f90' 1
 ! CHECK-NEXT: CookedSource::provenanceMap_:
 ! CHECK-NEXT: offsets [{{[0-9]*}}..{{[0-9]*}}] -> provenances [{{[0-9]*}}..{{[0-9]*}}] ({{[0-9]*}} bytes)
 ! CHECK-NEXT: CookedSource::invertedMap_:
 ! CHECK-NEXT: provenances [{{[0-9]*}}..{{[0-9]*}}] ({{[0-9]*}} bytes) -> offsets [{{[0-9]*}}..{{[0-9]*}}]
 ! CHECK-EMPTY:
 
 program A
 end

flang/test/Parser/line-directive.f90

+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s
+!CHECK: #line "{{.*[/\\]}}line-directive.f90" 3
+subroutine s
+  implicit none
+  a = 1.
+#line 100
+!CHECK: #line 101
+  b = 2.
+#line "sourceFile.cobol" 200
+!CHECK: #line "sourceFile.cobol" 201
+  c = 3.
+# 300
+!CHECK: #line 301
+  d = 4.
+# "/dev/random" 400
+!CHECK: #line "/dev/random" 401
+  e = 5.
+end

flang/test/Semantics/line-directive.f90

+!RUN: not %flang -fsyntax-only %s 2>&1 | FileCheck %s
+subroutine s
+  implicit none
+!CHECK: line-directive.f90:5:3: error: No explicit type declared for 'a'
+  a = 1.
+#line 100
+!CHECK: line-directive.f90:101:3: error: No explicit type declared for 'b'
+  b = 2.
+#line "sourceFile.cobol" 200
+!CHECK: sourceFile.cobol:201:3: error: No explicit type declared for 'c'
+  c = 3.
+# 300
+!CHECK: sourceFile.cobol:301:3: error: No explicit type declared for 'd'
+  d = 4.
+# "/dev/random" 400
+!CHECK: random:401:3: error: No explicit type declared for 'e'
+  e = 5.
+end