[VFS] Add property 'fallthrough' that controls fallback to real file system.

[vsapsai]

Summary

Default property value 'true' preserves current behavior. Value 'false'
can be used to create VFS "root", file system that gives better control
over which files compiler can use during compilation.

Non-fallthrough use case changes how we treat multiple VFS overlay
files. Instead of all of them being at the same level just above a real
file system, now they are nested and subsequent overlays can refer to
files in previous overlays.

rdar://problem/39465552

[vsapsai]

This draft is not ready yet and published to facilitate review of https://github.com/apple/swift/pull/18328

[vsapsai]

Draft is abandoned in favor of D50539.

clang/lib/Basic/VirtualFileSystem.cpp

public:
   VFSFromYamlDirIterImpl(const Twine &Path, RedirectingFileSystem &FS,
                          RedirectingDirectoryEntry::iterator Begin,
                          RedirectingDirectoryEntry::iterator End,
                          std::error_code &EC);
 
   std::error_code increment() override;
 };
 
+class CombinedFSDirIterImpl : public clang::vfs::detail::DirIterImpl {
+  FileSystem &ExtraFS;
+  std::string Path;
+  bool IsWorkingWithExtraFS;
+  directory_iterator CurrentDirIter;
+  llvm::StringSet<> SeenNames;
+
+  std::error_code incrementFS() {
+    // assert(!IsWorkingWithExtraFS && "incrementing past end");
+    if (IsWorkingWithExtraFS)
+      return {};
+    IsWorkingWithExtraFS = true;
+    std::error_code EC;
+    CurrentDirIter = (ExtraFS).dir_begin(Path, EC);
+    if (EC && EC != errc::no_such_file_or_directory)
+      return EC;
+    return {};
+  }
+
+  std::error_code incrementDirIter(bool IsFirstTime) {
+    assert((IsFirstTime || CurrentDirIter != directory_iterator()) &&
+           "incrementing past end");
+    std::error_code EC;
+    if (!IsFirstTime)
+      CurrentDirIter.increment(EC);
+    if (!EC && CurrentDirIter == directory_iterator())
+      EC = incrementFS();
+    return EC;
+  }
+
+  std::error_code incrementImpl(bool IsFirstTime) {
+    while (true) {
+      std::error_code EC = incrementDirIter(IsFirstTime);
+      if (EC || CurrentDirIter == directory_iterator()) {
+        CurrentEntry = Status();
+        return EC;
+      }
+      CurrentEntry = *CurrentDirIter;
+      StringRef Name = llvm::sys::path::filename(CurrentEntry.getName());
+      if (SeenNames.insert(Name).second)
+        return EC; // name not seen before
+    }
+    llvm_unreachable("returned above");
+  }
+
+public:
+  CombinedFSDirIterImpl(const Twine &Path, FileSystem &FS,
+                        directory_iterator DirIter, std::error_code &EC)
+      : ExtraFS(FS), Path(Path.str()), IsWorkingWithExtraFS(false),
+        CurrentDirIter(DirIter) {
+    EC = incrementImpl(true);
+  }
+
+  std::error_code increment() override { return incrementImpl(false); }
+};
+
 /// A virtual file system parsed from a YAML file.
 ///
 /// Currently, this class allows creating virtual directories and mapping
 /// virtual file paths to existing external files, available in \c ExternalFS.
 ///
 /// The basic structure of the parsed file is:
 /// \verbatim
 /// {
 ///   'version': <version number>,
 ///   <optional configuration>
 ///   'roots': [
 ///              <directory entries>
 ///            ]
 /// }
 /// \endverbatim
 ///
 /// All configuration options are optional.
 ///   'case-sensitive': <boolean, default=true>
 ///   'use-external-names': <boolean, default=true>
 ///   'overlay-relative': <boolean, default=false>
 ///   'ignore-non-existent-contents': <boolean, default=true>
+///   'fallthrough': <boolean, default=true>
 ///
 /// Virtual directories are represented as
 /// \verbatim
 /// {
 ///   'type': 'directory',
 ///   'name': <string>,
 ///   'contents': [ <file or directory entries> ]
 /// }
class RedirectingFileSystem : public vfs::FileSystem {
 
   /// Whether an invalid path obtained via 'external-contents' should
   /// cause iteration on the VFS to stop. If 'true', the VFS should ignore
   /// the entry and continue with the next. Allows YAML files to be shared
   /// across multiple compiler invocations regardless of prior existent
   /// paths in 'external-contents'. This global value is overridable on a
   /// per-file basis.
   bool IgnoreNonExistentContents = true;
+
+  /// Whether to attempt a file lookup in external file system after it wasn't
+  /// found in VFS.
+  bool IsFallthrough = true;
   /// @}
 
   /// Virtual file paths and external files could be canonicalized without "..",
   /// "." and "./" in their paths. FIXME: some unittests currently fail on
   /// win32 when using remove_dots and remove_leading_dotslash on paths.
   bool UseCanonicalizedPaths =
 #ifdef _WIN32
       false;
private:
 
   /// Get the status of a given an \c Entry.
   ErrorOr<Status> status(const Twine &Path, Entry *E);
 
 public:
   /// Looks up \p Path in \c Roots.
   ErrorOr<Entry *> lookupPath(const Twine &Path);
 
+  ErrorOr<Status> statusImpl(const Twine &Path);
+
   /// Parses \p Buffer, which is expected to be in YAML format and
   /// returns a virtual file system representing its contents.
   static RedirectingFileSystem *
   create(std::unique_ptr<MemoryBuffer> Buffer,
          SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
          void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
 
   ErrorOr<Status> status(const Twine &Path) override;
   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
 
   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
     return ExternalFS->getCurrentWorkingDirectory();
   }
 
   std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
     return ExternalFS->setCurrentWorkingDirectory(Path);
   }
 
-  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override{
+  directory_iterator dir_begin_impl(const Twine &Dir, std::error_code &EC) {
     ErrorOr<Entry *> E = lookupPath(Dir);
     if (!E) {
       EC = E.getError();
       return {};
     }
     ErrorOr<Status> S = status(Dir, *E);
     if (!S) {
       EC = S.getError();
       return {};
     }
     if (!S->isDirectory()) {
       EC = std::error_code(static_cast<int>(errc::not_a_directory),
                            std::system_category());
       return {};
     }
 
     auto *D = cast<RedirectingDirectoryEntry>(*E);
     return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir,
         *this, D->contents_begin(), D->contents_end(), EC));
   }
 
+  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override {
+    return directory_iterator(std::make_shared<CombinedFSDirIterImpl>(
+        Dir, *ExternalFS, dir_begin_impl(Dir, EC), EC));
+  }
+
   void setExternalContentsPrefixDir(StringRef PrefixDir) {
     ExternalContentsPrefixDir = PrefixDir.str();
   }
 
   StringRef getExternalContentsPrefixDir() const {
     return ExternalContentsPrefixDir;
   }
 
bool parse(yaml::Node *Root, RedirectingFileSystem *FS) {
     }
 
     KeyStatusPair Fields[] = {
       KeyStatusPair("version", true),
       KeyStatusPair("case-sensitive", false),
       KeyStatusPair("use-external-names", false),
       KeyStatusPair("overlay-relative", false),
       KeyStatusPair("ignore-non-existent-contents", false),
+      KeyStatusPair("fallthrough", false),
       KeyStatusPair("roots", true),
     };
 
     DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
     std::vector<std::unique_ptr<Entry>> RootEntries;
 
     // Parse configuration and 'roots'
     for (auto &I : *Top) {
for (auto &I : *Top) {
         if (!parseScalarBool(I.getValue(), FS->IsRelativeOverlay))
           return false;
       } else if (Key == "use-external-names") {
         if (!parseScalarBool(I.getValue(), FS->UseExternalNames))
           return false;
       } else if (Key == "ignore-non-existent-contents") {
         if (!parseScalarBool(I.getValue(), FS->IgnoreNonExistentContents))
           return false;
+      } else if (Key == "fallthrough") {
+        if (!parseScalarBool(I.getValue(), FS->IsFallthrough))
+          return false;
       } else {
         llvm_unreachable("key missing from Keys");
       }
     }
 
     if (Stream.failed())
       return false;
 
if (S)
                                      *S);
     return S;
   } else { // directory
     auto *DE = cast<RedirectingDirectoryEntry>(E);
     return Status::copyWithNewName(DE->getStatus(), Path.str());
   }
 }
 
-ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
+ErrorOr<Status> RedirectingFileSystem::statusImpl(const Twine &Path) {
   ErrorOr<Entry *> Result = lookupPath(Path);
   if (!Result)
     return Result.getError();
   return status(Path, *Result);
 }
 
+ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
+  ErrorOr<Status> Result = statusImpl(Path);
+  if (IsFallthrough && !Result &&
+      Result.getError() == llvm::errc::no_such_file_or_directory)
+    return ExternalFS->status(Path);
+  return Result;
+}
+
 namespace {
 
 /// Provide a file wrapper with an overriden status.
 class FileWithFixedStatus : public File {
   std::unique_ptr<File> InnerFile;
   Status S;
 
 public:
public:
   std::error_code close() override { return InnerFile->close(); }
 };
 
 } // namespace
 
 ErrorOr<std::unique_ptr<File>>
 RedirectingFileSystem::openFileForRead(const Twine &Path) {
   ErrorOr<Entry *> E = lookupPath(Path);
-  if (!E)
+  if (!E) {
+    if (IsFallthrough &&
+        E.getError() == llvm::errc::no_such_file_or_directory) {
+      return ExternalFS->openFileForRead(Path);
+    }
     return E.getError();
+  }
 
   auto *F = dyn_cast<RedirectingFileEntry>(*E);
   if (!F) // FIXME: errc::not_a_file?
     return make_error_code(llvm::errc::invalid_argument);
 
   auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath());
   if (!Result)
     return Result;

 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
     const Twine &_Path, RedirectingFileSystem &FS,
     RedirectingDirectoryEntry::iterator Begin,
     RedirectingDirectoryEntry::iterator End, std::error_code &EC)
     : Dir(_Path.str()), FS(FS), Current(Begin), End(End) {
   while (Current != End) {
     SmallString<128> PathStr(Dir);
     llvm::sys::path::append(PathStr, (*Current)->getName());
-    llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
+    llvm::ErrorOr<vfs::Status> S = FS.statusImpl(PathStr);
     if (S) {
       CurrentEntry = *S;
       return;
     }
     // Skip entries which do not map to a reliable external content.
     if (FS.ignoreNonExistentContents() &&
         S.getError() == llvm::errc::no_such_file_or_directory) {
       ++Current;
       continue;
     } else {
       EC = S.getError();
       break;
     }
   }
 }
 
 std::error_code VFSFromYamlDirIterImpl::increment() {
   assert(Current != End && "cannot iterate past end");
   while (++Current != End) {
     SmallString<128> PathStr(Dir);
     llvm::sys::path::append(PathStr, (*Current)->getName());
-    llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
+    llvm::ErrorOr<vfs::Status> S = FS.statusImpl(PathStr);
     if (!S) {
       // Skip entries which do not map to a reliable external content.
       if (FS.ignoreNonExistentContents() &&
           S.getError() == llvm::errc::no_such_file_or_directory) {
         continue;
       } else {
         return S.getError();
       }

clang/lib/Frontend/CompilerInvocation.cpp

 
 IntrusiveRefCntPtr<vfs::FileSystem>
 createVFSFromCompilerInvocation(const CompilerInvocation &CI,
                                 DiagnosticsEngine &Diags,
                                 IntrusiveRefCntPtr<vfs::FileSystem> BaseFS) {
   if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty())
     return BaseFS;
 
-  IntrusiveRefCntPtr<vfs::OverlayFileSystem> Overlay(
-      new vfs::OverlayFileSystem(BaseFS));
+  IntrusiveRefCntPtr<vfs::FileSystem> Result = BaseFS;
   // earlier vfs files are on the bottom
   for (const auto &File : CI.getHeaderSearchOpts().VFSOverlayFiles) {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
-        BaseFS->getBufferForFile(File);
+        Result->getBufferForFile(File);
     if (!Buffer) {
       Diags.Report(diag::err_missing_vfs_overlay_file) << File;
       continue;
     }
 
-    IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getVFSFromYAML(
-        std::move(Buffer.get()), /*DiagHandler*/ nullptr, File);
-    if (FS)
-      Overlay->pushOverlay(FS);
-    else
+    IntrusiveRefCntPtr<vfs::FileSystem> FS =
+        vfs::getVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr,
+                            File, /*DiagContext*/ nullptr, Result);
+    if (!FS) {
       Diags.Report(diag::err_invalid_vfs_overlay) << File;
+      continue;
+    }
+
+    Result = FS;
   }
-  return Overlay;
+  return Result;
 }
 
 } // namespace clang

clang/test/VFS/Inputs/vfsroot.yaml

+{
+  'version': 0,
+  'ignore-non-existent-contents': false,
+  'fallthrough': false,
+  'roots': [
+    { 'name': '/tests', 'type': 'directory',
+      'contents': [
+        { 'name': 'vfsroot-include.c', 'type': 'file',
+          'external-contents': 'TEST_DIR/vfsroot-include.c'
+        },
+        { 'name': 'vfsroot-with-overlay.c', 'type': 'file',
+          'external-contents': 'TEST_DIR/vfsroot-with-overlay.c'
+        }
+      ]
+    },
+    { 'name': '/direct-vfs-root-files', 'type': 'directory',
+      'contents': [
+        { 'name': 'not_real.h', 'type': 'file',
+          'external-contents': 'TEST_DIR/Inputs/actual_header.h'
+        },
+        { 'name': 'vfsoverlay.yaml', 'type': 'file',
+          'external-contents': 'TEMP_DIR/vfsoverlay.yaml'
+        }
+      ]
+    },
+    { 'name': '/indirect-vfs-root-files', 'type': 'directory',
+      'contents': [
+        { 'name': 'actual_header.h', 'type': 'file',
+          'external-contents': 'TEST_DIR/Inputs/actual_header.h'
+        }
+      ]
+    }
+  ]
+}

clang/test/VFS/vfsroot-include.c

+// REQUIRES: shell
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: sed -e "s:TEST_DIR:%S:g" %S/Inputs/vfsroot.yaml > %t.yaml
+// RUN: not %clang_cc1 -Werror -ivfsoverlay %t.yaml -I %S/Inputs -I /direct-vfs-root-files -fsyntax-only /tests/vfsroot-include.c 2>&1 | FileCheck %s
+// The line above tests that the compiler input file is looked up through VFS.
+
+// Test successful include through the VFS.
+#include "not_real.h"
+
+// Test that a file missing from the VFS root is not found, even if it is
+// discoverable through the real file system. Fatal error should be the last
+// in the file as it hides other errors.
+#include "actual_header.h"
+// CHECK: fatal error: 'actual_header.h' file not found
+// CHECK: 1 error generated.
+// CHECK-NOT: error

clang/test/VFS/vfsroot-with-overlay.c

+// REQUIRES: shell
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: sed -e "s:TEST_DIR:%S:g" -e "s:TEMP_DIR:%t:g" %S/Inputs/vfsroot.yaml > %t.yaml
+// RUN: sed -e "s:INPUT_DIR:/indirect-vfs-root-files:g" -e "s:OUT_DIR:/overlay-dir:g" %S/Inputs/vfsoverlay.yaml > %t/vfsoverlay.yaml
+// RUN: %clang_cc1 -Werror -ivfsoverlay %t.yaml -ivfsoverlay /direct-vfs-root-files/vfsoverlay.yaml -I /overlay-dir -fsyntax-only /tests/vfsroot-with-overlay.c
+
+#include "not_real.h"
+
+void foo() {
+  bar();
+}