[flang] Search for #include "file" in right directory

[klausler]

Summary

Make the #include "file" preprocessing directive begin its
search in the same directory as the file containing the directive,
as other preprocessors and our Fortran INCLUDE statement do.

Avoid current working directory for all source files after the original.

[awarzynski]

Hi @klausler, thank you for submitting this. This patch updates an API that's used in the new driver (i.e. updates in the new driver are missing):

• FrontendActions
• unittests

As most Flang buildbots build the new driver, this patch has introduced quite a few build failures:

• http://lab.llvm.org:8011/#/builders/32/builds/2642
• http://lab.llvm.org:8011/#/builders/33/builds/2131
• http://lab.llvm.org:8011/#/builders/135/builds/1473
• http://lab.llvm.org:8011/#/builders/66/builds/1559
• http://lab.llvm.org:8011/#/builders/134/builds/1409
• http://lab.llvm.org:8011/#/builders/132/builds/1817

EDIT
Fixing the build is rather straightforward:

diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index ee6f71cfc5af..925c7e64ed68 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -28,7 +28,7 @@ void InputOutputTestAction::ExecuteAction() {
   CompilerInstance &ci = instance();
   Fortran::parser::AllSources &allSources{ci.allSources()};
   const Fortran::parser::SourceFile *sf;
-  sf = allSources.Open(path, error_stream);
+  sf = allSources.Open(path, error_stream, std::nullopt);
   llvm::ArrayRef<char> fileContent = sf->content();

   // Output file descriptor to receive the content of input file.
diff --git a/flang/unittests/Frontend/CompilerInstanceTest.cpp b/flang/unittests/Frontend/CompilerInstanceTest.cpp
index b56473ea44e3..258213344363 100644
--- a/flang/unittests/Frontend/CompilerInstanceTest.cpp
+++ b/flang/unittests/Frontend/CompilerInstanceTest.cpp
@@ -47,7 +47,7 @@ TEST(CompilerInstance, SanityCheckForFileManager) {
   llvm::raw_string_ostream error_stream{buf};
   CompilerInstance compInst;
   const Fortran::parser::SourceFile *sf =
-      compInst.allSources().Open(testFilePath, error_stream);
+      compInst.allSources().Open(testFilePath, error_stream, std::nullopt);

   // 3. Verify the content of the input file
   // This is just a sanity check to make sure that CompilerInstance is capable

However, this patch changes the behavior of the fronted. This causes some of the driver tests to fail (in particular the tests introduced in https://reviews.llvm.org/D93453).

I reverted this as this is currently blocking further work in the new driver and makes our builders to fail. A fixing patch should fix both the failing build and the failing tests.

Please, could test with FLANG_BUILD_NEW_DRIVER set to ON? Also, for a fixing patch:

• could you add more reviewers?
• could you add tests?

Is there any way I can help way with this? Thank you!

flang/include/flang/Parser/provenance.h

public:
   std::size_t size() const { return range_.size(); }
   const char &operator[](Provenance) const;
   Encoding encoding() const { return encoding_; }
   AllSources &set_encoding(Encoding e) {
     encoding_ = e;
     return *this;
   }
 
-  void PushSearchPathDirectory(std::string);
-  std::string PopSearchPathDirectory();
-  const SourceFile *Open(std::string path, llvm::raw_ostream &error);
+  void AppendSearchPathDirectory(std::string); // new last directory
+  const SourceFile *Open(std::string path, llvm::raw_ostream &error,
+      const std::optional<std::string> &prependPath);
   const SourceFile *ReadStandardInput(llvm::raw_ostream &error);
 
   ProvenanceRange AddIncludedFile(
       const SourceFile &, ProvenanceRange, bool isModule = false);
   ProvenanceRange AddMacroCall(
       ProvenanceRange def, ProvenanceRange use, const std::string &expansion);
   ProvenanceRange AddCompilerInsertion(std::string);
 
private:
 
   const Origin &MapToOrigin(Provenance) const;
 
   // Elements are in ascending & contiguous order of .covers.
   std::vector<Origin> origin_;
   ProvenanceRange range_;
   std::map<char, Provenance> compilerInsertionProvenance_;
   std::vector<std::unique_ptr<SourceFile>> ownedSourceFiles_;
-  std::vector<std::string> searchPath_;
+  std::list<std::string> searchPath_;
   Encoding encoding_{Encoding::UTF_8};
 };
 
 // Represents the result of preprocessing and prescanning a single source
 // file (and all its inclusions) or module file.  Parsers operate within
 // single instances of CookedSource.
 class CookedSource {
 public:

flang/include/flang/Parser/source.h

 // Source file content is lightly normalized when the file is read.
 //  - Line ending markers are converted to single newline characters
 //  - A newline character is added to the last line of the file if one is needed
 //  - A Unicode byte order mark is recognized if present.
 
 #include "characters.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstddef>
+#include <list>
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
 
 namespace llvm {
 class raw_ostream;
 }
 
 namespace Fortran::parser {
 
 std::string DirectoryName(std::string path);
-std::string LocateSourceFile(
-    std::string name, const std::vector<std::string> &searchPath);
+std::optional<std::string> LocateSourceFile(
+    std::string name, const std::list<std::string> &searchPath);
 
 class SourceFile;
 
 struct SourcePosition {
   const SourceFile &file;
   int line, column;
 };
 

flang/lib/Parser/parsing.cpp

 Parsing::Parsing(AllCookedSources &allCooked) : allCooked_{allCooked} {}
 Parsing::~Parsing() {}
 
 const SourceFile *Parsing::Prescan(const std::string &path, Options options) {
   options_ = options;
   AllSources &allSources{allCooked_.allSources()};
   if (options.isModuleFile) {
     for (const auto &path : options.searchDirectories) {
-      allSources.PushSearchPathDirectory(path);
+      allSources.AppendSearchPathDirectory(path);
     }
   }
 
   std::string buf;
   llvm::raw_string_ostream fileError{buf};
   const SourceFile *sourceFile;
   if (path == "-") {
     sourceFile = allSources.ReadStandardInput(fileError);
   } else {
-    sourceFile = allSources.Open(path, fileError);
+    std::optional<std::string> currentDirectory{"."};
+    sourceFile = allSources.Open(path, fileError, currentDirectory);
   }
   if (!fileError.str().empty()) {
     ProvenanceRange range{allSources.AddCompilerInsertion(path)};
     messages_.Say(range, "%s"_err_en_US, fileError.str());
     return sourceFile;
   }
   CHECK(sourceFile);
 
   if (!options.isModuleFile) {
     // For .mod files we always want to look in the search directories.
-    // For normal source files we don't push them until after the primary
+    // For normal source files we don't add them until after the primary
     // source file has been opened.  If foo.f is missing from the current
     // working directory, we don't want to accidentally read another foo.f
     // from another directory that's on the search path.
     for (const auto &path : options.searchDirectories) {
-      allSources.PushSearchPathDirectory(path);
+      allSources.AppendSearchPathDirectory(path);
     }
   }
 
   Preprocessor preprocessor{allSources};
   for (const auto &predef : options.predefinitions) {
     if (predef.second) {
       preprocessor.Define(predef.first, *predef.second);
     } else {

flang/lib/Parser/preprocessor.cpp

 }
 
 void Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) {
   std::size_t tokens{dir.SizeInTokens()};
   std::size_t j{dir.SkipBlanks(0)};
   if (j == tokens) {
     return;
   }
+  CHECK(prescanner); // TODO: change to reference
   if (dir.TokenAt(j).ToString() != "#") {
     prescanner->Say(dir.GetTokenProvenanceRange(j), "missing '#'"_err_en_US);
     return;
   }
   j = dir.SkipBlanks(j + 1);
   while (tokens > 0 && dir.TokenAt(tokens - 1).IsBlank()) {
     --tokens;
   }
void Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) {
   } else if (dirName == "include") {
     if (j == tokens) {
       prescanner->Say(
           dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
           "#include: missing name of file to include"_err_en_US);
       return;
     }
     std::string include;
+    std::optional<std::string> prependPath;
     if (dir.TokenAt(j).ToString() == "<") { // #include <foo>
       std::size_t k{j + 1};
       if (k >= tokens) {
         prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
             "#include: file name missing"_err_en_US);
         return;
       }
       while (k < tokens && dir.TokenAt(k) != ">") {
         ++k;
       }
       if (k >= tokens) {
         prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
             "#include: expected '>' at end of included file"_en_US);
       }
       TokenSequence braced{dir, j + 1, k - j - 1};
       include = ReplaceMacros(braced, *prescanner).ToString();
       j = k;
     } else if ((include = dir.TokenAt(j).ToString()).substr(0, 1) == "\"" &&
         include.substr(include.size() - 1, 1) == "\"") { // #include "foo"
       include = include.substr(1, include.size() - 2);
+      // #include "foo" starts search in directory of file containing
+      // the directive
+      auto prov{dir.GetTokenProvenanceRange(dirOffset).start()};
+      if (const auto *currentFile{allSources_.GetSourceFile(prov)}) {
+        prependPath = DirectoryName(currentFile->path());
+      }
     } else {
       prescanner->Say(dir.GetTokenProvenanceRange(j < tokens ? j : tokens - 1),
           "#include: expected name of file to include"_err_en_US);
       return;
     }
     if (include.empty()) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#include: empty include file name"_err_en_US);
       return;
     }
     j = dir.SkipBlanks(j + 1);
     if (j < tokens && dir.TokenAt(j).ToString() != "!") {
       prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
           "#include: extra stuff ignored after file name"_en_US);
     }
     std::string buf;
     llvm::raw_string_ostream error{buf};
-    const SourceFile *included{allSources_.Open(include, error)};
+    const SourceFile *included{allSources_.Open(include, error, prependPath)};
     if (!included) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#include: %s"_err_en_US, error.str());
     } else if (included->bytes() > 0) {
       ProvenanceRange fileRange{
           allSources_.AddIncludedFile(*included, dir.GetProvenanceRange())};
       Prescanner{*prescanner}
           .set_encoding(included->encoding())

flang/lib/Parser/prescan.cpp

if (*p != '\n' && *p != '!') {
     for (; *p != '\n' && *p != '!'; ++p) {
     }
     Say(GetProvenanceRange(garbage, p),
         "excess characters after path name"_en_US);
   }
   std::string buf;
   llvm::raw_string_ostream error{buf};
   Provenance provenance{GetProvenance(nextLine_)};
-  const SourceFile *currentFile{allSources_.GetSourceFile(provenance)};
-  if (currentFile) {
-    allSources_.PushSearchPathDirectory(DirectoryName(currentFile->path()));
-  }
-  const SourceFile *included{allSources_.Open(path, error)};
-  if (currentFile) {
-    allSources_.PopSearchPathDirectory();
+  std::optional<std::string> prependPath;
+  if (const SourceFile * currentFile{allSources_.GetSourceFile(provenance)}) {
+    prependPath = DirectoryName(currentFile->path());
   }
+  const SourceFile *included{allSources_.Open(path, error, prependPath)};
   if (!included) {
     Say(provenance, "INCLUDE: %s"_err_en_US, error.str());
   } else if (included->bytes() > 0) {
     ProvenanceRange includeLineRange{
         provenance, static_cast<std::size_t>(p - nextLine_)};
     ProvenanceRange fileRange{
         allSources_.AddIncludedFile(*included, includeLineRange)};
     Prescanner{*this}.set_encoding(included->encoding()).Prescan(fileRange);

flang/lib/Parser/provenance.cpp

 
 AllSources::~AllSources() {}
 
 const char &AllSources::operator[](Provenance at) const {
   const Origin &origin{MapToOrigin(at)};
   return origin[origin.covers.MemberOffset(at)];
 }
 
-void AllSources::PushSearchPathDirectory(std::string directory) {
+void AllSources::AppendSearchPathDirectory(std::string directory) {
   // gfortran and ifort append to current path, PGI prepends
   searchPath_.push_back(directory);
 }
 
-std::string AllSources::PopSearchPathDirectory() {
-  std::string directory{searchPath_.back()};
-  searchPath_.pop_back();
-  return directory;
-}
-
-const SourceFile *AllSources::Open(std::string path, llvm::raw_ostream &error) {
+const SourceFile *AllSources::Open(std::string path, llvm::raw_ostream &error,
+    const std::optional<std::string> &prependPath) {
   std::unique_ptr<SourceFile> source{std::make_unique<SourceFile>(encoding_)};
-  if (source->Open(LocateSourceFile(path, searchPath_), error)) {
+  if (prependPath) {
+    // Set to "." for the initial source file; set to the directory name
+    // of the including file for #include "quoted-file" directives &
+    // INCLUDE statements.
+    searchPath_.push_front(*prependPath);
+  }
+  std::optional<std::string> found{LocateSourceFile(path, searchPath_)};
+  if (prependPath) {
+    searchPath_.pop_front();
+  }
+  if (!found) {
+    error << "Source file '" << path << "' was not found";
+    return nullptr;
+  } else if (source->Open(*found, error)) {
     return ownedSourceFiles_.emplace_back(std::move(source)).get();
   } else {
     return nullptr;
   }
 }
 
 const SourceFile *AllSources::ReadStandardInput(llvm::raw_ostream &error) {
   std::unique_ptr<SourceFile> source{std::make_unique<SourceFile>(encoding_)};

flang/lib/Parser/source.cpp

 }
 
 std::string DirectoryName(std::string path) {
   llvm::SmallString<128> pathBuf{path};
   llvm::sys::path::remove_filename(pathBuf);
   return pathBuf.str().str();
 }
 
-std::string LocateSourceFile(
-    std::string name, const std::vector<std::string> &searchPath) {
-  if (name.empty() || name == "-" || llvm::sys::path::is_absolute(name)) {
+std::optional<std::string> LocateSourceFile(
+    std::string name, const std::list<std::string> &searchPath) {
+  if (name == "-" || llvm::sys::path::is_absolute(name)) {
     return name;
   }
   for (const std::string &dir : searchPath) {
     llvm::SmallString<128> path{dir};
     llvm::sys::path::append(path, name);
     bool isDir{false};
     auto er = llvm::sys::fs::is_directory(path, isDir);
     if (!er && !isDir) {
       return path.str().str();
     }
   }
-  return name;
+  return std::nullopt;
 }
 
 std::size_t RemoveCarriageReturns(llvm::MutableArrayRef<char> buf) {
   std::size_t wrote{0};
   char *buffer{buf.data()};
   char *p{buf.data()};
   std::size_t bytes = buf.size();
   while (bytes > 0) {
bool SourceFile::Open(std::string path, llvm::raw_ostream &error) {
   buf_ = std::move(bufOr.get());
   ReadFile();
   return true;
 }
 
 bool SourceFile::ReadStandardInput(llvm::raw_ostream &error) {
   Close();
   path_ = "standard input";
-
   auto buf_or = llvm::MemoryBuffer::getSTDIN();
   if (!buf_or) {
     auto err = buf_or.getError();
     error << err.message();
     return false;
   }
   auto inbuf = std::move(buf_or.get());
   buf_ =
       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(inbuf->getBufferSize());
   llvm::copy(inbuf->getBuffer(), buf_->getBufferStart());
   ReadFile();
   return true;
 }
 
 void SourceFile::ReadFile() {
   buf_end_ = RemoveCarriageReturns(buf_->getBuffer());
   if (content().size() == 0 || content().back() != '\n') {
     // Don't bother to copy if we have spare memory
     if (content().size() >= buf_->getBufferSize()) {
       auto tmp_buf{llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
           content().size() + 1)};
       llvm::copy(content(), tmp_buf->getBufferStart());
-      Close();
       buf_ = std::move(tmp_buf);
     }
     buf_end_++;
     buf_->getBuffer()[buf_end_ - 1] = '\n';
   }
   IdentifyPayload();
   RecordLineStarts();
 }

flang/tools/f18/f18.cpp

struct GetDefinitionArgs {
   int line, startColumn, endColumn;
 };
 
 struct DriverOptions {
   DriverOptions() {}
   bool verbose{false}; // -v
   bool compileOnly{false}; // -c
   std::string outputPath; // -o path
-  std::vector<std::string> searchDirectories{"."s}; // -I dir
+  std::vector<std::string> searchDirectories; // -I dir
   std::string moduleDirectory{"."s}; // -module dir
   std::string moduleFileSuffix{".mod"}; // -moduleSuffix suff
   bool forcedForm{false}; // -Mfixed or -Mfree appeared
   bool warnOnNonstandardUsage{false}; // -Mstandard
   bool warningsAreErrors{false}; // -Werror
   bool byteswapio{false}; // -byteswapio
   Fortran::parser::Encoding encoding{Fortran::parser::Encoding::UTF_8};
   bool parseOnly{false};