[flang] Search for #include "file" in right directory (take 2)

[klausler]

Summary

Make the #include "file" preprocessing directive begin its
search in the same directory as the file containing the directive,
as other preprocessors and our Fortran INCLUDE statement do.

Avoid current working directory for all source files except the original.

Resolve tests.

[kiranchandramohan]

flang/test/Flang-Driver/Inputs/basictestmoduleone.mod

1

I am guessing this version number (v1) might need to be updated with a change in module format. Is that expected to be rare since f18 uses the parser to parse the module file?

[awarzynski]

Thank you for re-submitting this so quickly @klausler !

These changes look good to me. I have one question about module files - is it safe to assume that modules with version information v1 are always going to be supported? I'm concerned that when we bump the version then we will also have to update the uploaded module files.

flang/test/Flang-Driver/include-module.f90

21

[nit] This and the line above are mutually exclusive, right? So this line could safely be deleted. Or replaced with:

! SINGLEINCLUDE:Source file 'basictestmoduletwo.mod' was not found

flang/include/flang/Parser/provenance.h

     return *this;
   }
 
-  void PushSearchPathDirectory(std::string);
-  std::string PopSearchPathDirectory();
-  const SourceFile *Open(std::string path, llvm::raw_ostream &error);
+  void AppendSearchPathDirectory(std::string); // new last directory
+  const SourceFile *Open(std::string path, llvm::raw_ostream &error,
+      std::optional<std::string> &&prependPath = std::nullopt);
   const SourceFile *ReadStandardInput(llvm::raw_ostream &error);
 
   ProvenanceRange AddIncludedFile(

   ProvenanceRange range_;
   std::map<char, Provenance> compilerInsertionProvenance_;
   std::vector<std::unique_ptr<SourceFile>> ownedSourceFiles_;
-  std::vector<std::string> searchPath_;
+  std::list<std::string> searchPath_;
   Encoding encoding_{Encoding::UTF_8};
 };
 

flang/include/flang/Parser/source.h

 #include "characters.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstddef>
+#include <list>
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>

 namespace Fortran::parser {
 
 std::string DirectoryName(std::string path);
-std::string LocateSourceFile(
-    std::string name, const std::vector<std::string> &searchPath);
+std::optional<std::string> LocateSourceFile(
+    std::string name, const std::list<std::string> &searchPath);
 
 class SourceFile;
 

flang/lib/Frontend/FrontendActions.cpp

   CompilerInstance &ci = instance();
   Fortran::parser::AllSources &allSources{ci.allSources()};
   const Fortran::parser::SourceFile *sf;
-  sf = allSources.Open(path, error_stream);
+  sf = allSources.Open(path, error_stream, std::optional<std::string>{"."s});
   llvm::ArrayRef<char> fileContent = sf->content();
 
   // Output file descriptor to receive the content of input file.

flang/lib/Parser/parsing.cpp

   AllSources &allSources{allCooked_.allSources()};
   if (options.isModuleFile) {
     for (const auto &path : options.searchDirectories) {
-      allSources.PushSearchPathDirectory(path);
+      allSources.AppendSearchPathDirectory(path);
     }
   }
 

   if (path == "-") {
     sourceFile = allSources.ReadStandardInput(fileError);
   } else {
-    sourceFile = allSources.Open(path, fileError);
+    std::optional<std::string> currentDirectory{"."};
+    sourceFile = allSources.Open(path, fileError, std::move(currentDirectory));
   }
   if (!fileError.str().empty()) {
     ProvenanceRange range{allSources.AddCompilerInsertion(path)};

 
   if (!options.isModuleFile) {
     // For .mod files we always want to look in the search directories.
-    // For normal source files we don't push them until after the primary
+    // For normal source files we don't add them until after the primary
     // source file has been opened.  If foo.f is missing from the current
     // working directory, we don't want to accidentally read another foo.f
     // from another directory that's on the search path.
     for (const auto &path : options.searchDirectories) {
-      allSources.PushSearchPathDirectory(path);
+      allSources.AppendSearchPathDirectory(path);
     }
   }
 

flang/lib/Parser/preprocessor.cpp

   if (j == tokens) {
     return;
   }
+  CHECK(prescanner); // TODO: change to reference
   if (dir.TokenAt(j).ToString() != "#") {
     prescanner->Say(dir.GetTokenProvenanceRange(j), "missing '#'"_err_en_US);
     return;

       return;
     }
     std::string include;
+    std::optional<std::string> prependPath;
     if (dir.TokenAt(j).ToString() == "<") { // #include <foo>
       std::size_t k{j + 1};
       if (k >= tokens) {

     } else if ((include = dir.TokenAt(j).ToString()).substr(0, 1) == "\"" &&
         include.substr(include.size() - 1, 1) == "\"") { // #include "foo"
       include = include.substr(1, include.size() - 2);
+      // #include "foo" starts search in directory of file containing
+      // the directive
+      auto prov{dir.GetTokenProvenanceRange(dirOffset).start()};
+      if (const auto *currentFile{allSources_.GetSourceFile(prov)}) {
+        prependPath = DirectoryName(currentFile->path());
+      }
     } else {
       prescanner->Say(dir.GetTokenProvenanceRange(j < tokens ? j : tokens - 1),
           "#include: expected name of file to include"_err_en_US);

     }
     std::string buf;
     llvm::raw_string_ostream error{buf};
-    const SourceFile *included{allSources_.Open(include, error)};
+    const SourceFile *included{
+        allSources_.Open(include, error, std::move(prependPath))};
     if (!included) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#include: %s"_err_en_US, error.str());

flang/lib/Parser/prescan.cpp

   std::string buf;
   llvm::raw_string_ostream error{buf};
   Provenance provenance{GetProvenance(nextLine_)};
-  const SourceFile *currentFile{allSources_.GetSourceFile(provenance)};
-  if (currentFile) {
-    allSources_.PushSearchPathDirectory(DirectoryName(currentFile->path()));
-  }
-  const SourceFile *included{allSources_.Open(path, error)};
-  if (currentFile) {
-    allSources_.PopSearchPathDirectory();
+  std::optional<std::string> prependPath;
+  if (const SourceFile * currentFile{allSources_.GetSourceFile(provenance)}) {
+    prependPath = DirectoryName(currentFile->path());
   }
+  const SourceFile *included{
+      allSources_.Open(path, error, std::move(prependPath))};
   if (!included) {
     Say(provenance, "INCLUDE: %s"_err_en_US, error.str());
   } else if (included->bytes() > 0) {

flang/lib/Parser/provenance.cpp

   return origin[origin.covers.MemberOffset(at)];
 }
 
-void AllSources::PushSearchPathDirectory(std::string directory) {
+void AllSources::AppendSearchPathDirectory(std::string directory) {
   // gfortran and ifort append to current path, PGI prepends
   searchPath_.push_back(directory);
 }
 
-std::string AllSources::PopSearchPathDirectory() {
-  std::string directory{searchPath_.back()};
-  searchPath_.pop_back();
-  return directory;
-}
-
-const SourceFile *AllSources::Open(std::string path, llvm::raw_ostream &error) {
+const SourceFile *AllSources::Open(std::string path, llvm::raw_ostream &error,
+    std::optional<std::string> &&prependPath) {
   std::unique_ptr<SourceFile> source{std::make_unique<SourceFile>(encoding_)};
-  if (source->Open(LocateSourceFile(path, searchPath_), error)) {
+  if (prependPath) {
+    // Set to "." for the initial source file; set to the directory name
+    // of the including file for #include "quoted-file" directives &
+    // INCLUDE statements.
+    searchPath_.emplace_front(std::move(*prependPath));
+  }
+  std::optional<std::string> found{LocateSourceFile(path, searchPath_)};
+  if (prependPath) {
+    searchPath_.pop_front();
+  }
+  if (!found) {
+    error << "Source file '" << path << "' was not found";
+    return nullptr;
+  } else if (source->Open(*found, error)) {
     return ownedSourceFiles_.emplace_back(std::move(source)).get();
   } else {
     return nullptr;

flang/lib/Parser/source.cpp

   return pathBuf.str().str();
 }
 
-std::string LocateSourceFile(
-    std::string name, const std::vector<std::string> &searchPath) {
-  if (name.empty() || name == "-" || llvm::sys::path::is_absolute(name)) {
+std::optional<std::string> LocateSourceFile(
+    std::string name, const std::list<std::string> &searchPath) {
+  if (name == "-" || llvm::sys::path::is_absolute(name)) {
     return name;
   }
   for (const std::string &dir : searchPath) {

       return path.str().str();
     }
   }
-  return name;
+  return std::nullopt;
 }
 
 std::size_t RemoveCarriageReturns(llvm::MutableArrayRef<char> buf) {

 bool SourceFile::ReadStandardInput(llvm::raw_ostream &error) {
   Close();
   path_ = "standard input";
-
   auto buf_or = llvm::MemoryBuffer::getSTDIN();
   if (!buf_or) {
     auto err = buf_or.getError();

       auto tmp_buf{llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
           content().size() + 1)};
       llvm::copy(content(), tmp_buf->getBufferStart());
-      Close();
       buf_ = std::move(tmp_buf);
     }
     buf_end_++;

flang/test/Flang-Driver/Inputs/basictestmoduleone.mod

-NOT A REAL MODULE FILE - USE THIS ONLY FOR TESTING THE DRIVER
+!mod$ v1 sum:1f5a35ada852dc66
+module basictestmoduleone
+type::t1
+end type
+end

flang/test/Flang-Driver/Inputs/module-dir/basictestmoduleone.mod

+!mod$ v1 sum:449b70509dd4bce3
+module basictestmoduleone
+type::t2
+end type
+end

flang/test/Flang-Driver/Inputs/module-dir/basictestmoduletwo.mod

-NOT A REAL MODULE FILE - USE THIS ONLY FOR TESTING THE DRIVER
+!mod$ v1 sum:563b9a1f049282d2
+module basictestmoduletwo
+end

flang/test/Flang-Driver/include-header.f90

 !--------------------------------------------
 ! EXPECTED OUTPUT FOR MISSING INCLUDED FILE
 !--------------------------------------------
-! UNINCLUDED:No such file or directory
+! UNINCLUDED:#include: Source file 'basic-header-one.h' was not found
 ! UNINCLUDED-NOT:program b
 ! UNINCLUDED-NOT:program c
 

flang/test/Flang-Driver/include-module.f90

 ! Ensure argument -I works as expected with module files.
-! The module files for this test are not real module files.
 
 ! REQUIRES: new-flang-driver
 

 !-----------------------------------------
 ! EXPECTED OUTPUT FOR MISSING MODULE FILE
 !-----------------------------------------
-! SINGLEINCLUDE:No such file or directory
+! SINGLEINCLUDE:Error reading module file for module 'basictestmoduletwo'
 ! SINGLEINCLUDE-NOT:Error reading module file for module 'basictestmoduletwo'
+! SINGLEINCLUDE-NOT:error: Derived type 't1' not found
+! SINGLEINCLUDE:error: Derived type 't2' not found
 
 !---------------------------------------
 ! EXPECTED OUTPUT FOR ALL MODULES FOUND
 !---------------------------------------
-! INCLUDED-NOT:No such file or directory
+! INCLUDED-NOT:Error reading module file
+! INCLUDED-NOT:error: Derived type 't1' not found
+! INCLUDED:error: Derived type 't2' not found
 
 program test_dash_I_with_mod_files
     USE basictestmoduleone
     USE basictestmoduletwo
+    type(t1) :: x1 ! t1 defined in Inputs/basictestmoduleone.mod
+    type(t2) :: x2 ! t2 defined in Inputs/module-dir/basictestmoduleone.mod
 end

flang/tools/f18/f18.cpp

   bool verbose{false}; // -v
   bool compileOnly{false}; // -c
   std::string outputPath; // -o path
-  std::vector<std::string> searchDirectories{"."s}; // -I dir
+  std::vector<std::string> searchDirectories; // -I dir
   std::string moduleDirectory{"."s}; // -module dir
   std::string moduleFileSuffix{".mod"}; // -moduleSuffix suff
   bool forcedForm{false}; // -Mfixed or -Mfree appeared

flang/unittests/Frontend/FrontendActionTest.cpp

   EXPECT_TRUE(!outputDiagBuffer.empty());
   EXPECT_TRUE(
       llvm::StringRef(outputDiagBuffer.data())
-          .startswith(
+          .contains(
               ":1:14: error: IF statement is not allowed in IF statement\n"));
 }
 } // namespace