diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp --- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp +++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp @@ -363,26 +363,26 @@ std::move(OverrideOptions), std::move(FS)); } -llvm::IntrusiveRefCntPtr -getVfsFromFile(const std::string &OverlayFile, - llvm::IntrusiveRefCntPtr BaseFS) { - llvm::ErrorOr> Buffer = - BaseFS->getBufferForFile(OverlayFile); - if (!Buffer) { - llvm::errs() << "Can't load virtual filesystem overlay file '" - << OverlayFile << "': " << Buffer.getError().message() - << ".\n"; - return nullptr; - } - - IntrusiveRefCntPtr FS = vfs::getVFSFromYAML( - std::move(Buffer.get()), /*DiagHandler*/ nullptr, OverlayFile); - if (!FS) { - llvm::errs() << "Error: invalid virtual filesystem overlay file '" - << OverlayFile << "'.\n"; +static IntrusiveRefCntPtr +getVFSFromFile(StringRef OverlayFile) { + if (OverlayFile.empty()) + return makeIntrusiveRefCnt( + vfs::getRealFileSystem()); + + auto OverlayFS = vfs::getVFSFromYAMLs(OverlayFile, vfs::getRealFileSystem()); + if (auto Err = OverlayFS.takeError()) { + handleAllErrors(std::move(Err), [](const FileError &FE) { + if (FE.convertToErrorCode() == std::errc::no_such_file_or_directory) { + errs() << "Can't load virtual filesystem overlay file '" + << FE.getFileName() << "': " << FE.message() << ".\n"; + } else { + errs() << "Error: invalid virtual filesystem overlay file '" + << FE.getFileName() << "'.\n"; + } + }); return nullptr; } - return FS; + return std::move(*OverlayFS); } int clangTidyMain(int argc, const char **argv) { @@ -400,16 +400,10 @@ return 1; } - llvm::IntrusiveRefCntPtr BaseFS( - new vfs::OverlayFileSystem(vfs::getRealFileSystem())); - - if (!VfsOverlay.empty()) { - IntrusiveRefCntPtr VfsFromFile = - getVfsFromFile(VfsOverlay, BaseFS); - if (!VfsFromFile) - return 1; - BaseFS->pushOverlay(std::move(VfsFromFile)); - } + llvm::IntrusiveRefCntPtr BaseFS = + getVFSFromFile(VfsOverlay); + if (!BaseFS) + return 1; auto OwningOptionsProvider = createOptionsProvider(BaseFS); auto *OptionsProvider = OwningOptionsProvider.get(); diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -4693,28 +4693,22 @@ clang::createVFSFromCompilerInvocation( const CompilerInvocation &CI, DiagnosticsEngine &Diags, IntrusiveRefCntPtr BaseFS) { - if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty()) + ArrayRef OptFiles = CI.getHeaderSearchOpts().VFSOverlayFiles; + if (OptFiles.empty()) return BaseFS; - IntrusiveRefCntPtr Result = BaseFS; - // earlier vfs files are on the bottom - for (const auto &File : CI.getHeaderSearchOpts().VFSOverlayFiles) { - llvm::ErrorOr> Buffer = - Result->getBufferForFile(File); - if (!Buffer) { - Diags.Report(diag::err_missing_vfs_overlay_file) << File; - continue; - } - - IntrusiveRefCntPtr FS = llvm::vfs::getVFSFromYAML( - std::move(Buffer.get()), /*DiagHandler*/ nullptr, File, - /*DiagContext*/ nullptr, Result); - if (!FS) { - Diags.Report(diag::err_invalid_vfs_overlay) << File; - continue; - } - - Result = FS; + SmallVector Files(OptFiles.begin(), OptFiles.end()); + Expected> OverlayFS = + llvm::vfs::getVFSFromYAMLs(Files, BaseFS); + if (auto Err = OverlayFS.takeError()) { + llvm::handleAllErrors(std::move(Err), [&](const llvm::FileError &FE) { + if (FE.convertToErrorCode() == std::errc::no_such_file_or_directory) { + Diags.Report(diag::err_missing_vfs_overlay_file) << FE.getFileName(); + } else { + Diags.Report(diag::err_invalid_vfs_overlay) << FE.getFileName(); + } + }); + return BaseFS; } - return Result; + return *OverlayFS; } diff --git a/clang/test/VFS/Inputs/vfsroot.yaml b/clang/test/VFS/Inputs/vfsroot.yaml --- a/clang/test/VFS/Inputs/vfsroot.yaml +++ b/clang/test/VFS/Inputs/vfsroot.yaml @@ -26,13 +26,6 @@ } ] }, - { 'name': '/indirect-vfs-root-files', 'type': 'directory', - 'contents': [ - { 'name': 'actual_header.h', 'type': 'file', - 'external-contents': 'TEST_DIR/Inputs/actual_header.h' - } - ] - }, { 'name': 'TEST_DIR/Inputs/Broken.framework', 'type': 'directory', 'contents': [ { 'name': 'Headers/A.h', 'type': 'file', diff --git a/clang/test/VFS/directory.c b/clang/test/VFS/directory.c --- a/clang/test/VFS/directory.c +++ b/clang/test/VFS/directory.c @@ -1,13 +1,11 @@ // RUN: rm -rf %t // RUN: mkdir -p %t/Underlying // RUN: mkdir -p %t/Overlay -// RUN: mkdir -p %t/Middle // RUN: echo '// B.h in Underlying' > %t/Underlying/B.h // RUN: echo '#ifdef NESTED' >> %t/Underlying/B.h // RUN: echo '#include "C.h"' >> %t/Underlying/B.h // RUN: echo '#endif' >> %t/Underlying/B.h // RUN: echo '// C.h in Underlying' > %t/Underlying/C.h -// RUN: echo '// C.h in Middle' > %t/Middle/C.h // RUN: echo '// C.h in Overlay' > %t/Overlay/C.h // 1) Underlying -> Overlay (C.h found, B.h falling back to Underlying) @@ -17,31 +15,11 @@ // RUN: sed -e "s@INPUT_DIR@Overlay@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Underlying@g" %S/Inputs/vfsoverlay-directory-relative.yaml > %t/vfs-relative.yaml // RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs-relative.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s +// DIRECT: # 1 "{{.*(/|\\\\)Underlying(/|\\\\)}}B.h" // DIRECT: {{^}}// B.h in Underlying +// DIRECT: # 1 "{{.*(/|\\\\)Overlay(/|\\\\)}}C.h" // DIRECT: {{^}}// C.h in Overlay -// 2) Underlying -> Middle -> Overlay (C.h found, B.h falling back to Underlying) -// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}/Overlay@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Middle@g" %S/Inputs/vfsoverlay-directory.yaml > %t/vfs.yaml -// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}/Middle@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Underlying@g" %S/Inputs/vfsoverlay-directory.yaml > %t/vfs2.yaml -// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s -// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -DNESTED -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s - -// Same as direct above - -// 3) Underlying -> Middle -> Overlay (C.h falling back to Middle, B.h falling back to Underlying) -// RUN: rm -f %t/Overlay/C.h -// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=FALLBACK %s - -// FALLBACK: {{^}}// B.h in Underlying -// FALLBACK: {{^}}// C.h in Middle - -// 3) Underlying -> Middle -> Overlay (C.h falling back to Underlying, B.h falling back to Underlying) -// RUN: rm -f %t/Middle/C.h -// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=FALLBACK2 %s - -// FALLBACK2: {{^}}// B.h in Underlying -// FALLBACK2: {{^}}// C.h in Underlying - #include "B.h" #ifndef NESTED #include "C.h" diff --git a/clang/test/VFS/multiple-overlays.c b/clang/test/VFS/multiple-overlays.c new file mode 100644 --- /dev/null +++ b/clang/test/VFS/multiple-overlays.c @@ -0,0 +1,39 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s@EXTERNAL_DIR@%{/t:regex_replacement}/B@g" -e "s@NAME_DIR@%{/t:regex_replacement}/A@g" %t/vfs/base.yaml > %t/vfs/a-b.yaml +// RUN: sed -e "s@EXTERNAL_DIR@%{/t:regex_replacement}/C@g" -e "s@NAME_DIR@%{/t:regex_replacement}/B@g" %t/vfs/base.yaml > %t/vfs/b-c.yaml + +// Overlays should not be transitive, ie. given overlays of A -> B and B -> C, A should not remap to +// C. + +// RUN: %clang_cc1 -Werror -I %t/A -ivfsoverlay %t/vfs/b-c.yaml -ivfsoverlay %t/vfs/a-b.yaml -fsyntax-only -E -C %t/main.c 2>&1 | FileCheck --check-prefix=FROM_B %s +// FROM_B: # 1 "{{.*(/|\\\\)B(/|\\\\)}}Header.h" +// FROM_B: // Header.h in B + +// RUN: %clang_cc1 -Werror -I %t/B -ivfsoverlay %t/vfs/b-c.yaml -ivfsoverlay %t/vfs/a-b.yaml -fsyntax-only -E -C %t/main.c 2>&1 | FileCheck --check-prefix=FROM_C %s +// FROM_C: # 1 "{{.*(/|\\\\)C(/|\\\\)}}Header.h" +// FROM_C: // Header.h in C + +//--- main.c +#include "Header.h" + +//--- A/Header.h +// Header.h in A + +//--- B/Header.h +// Header.h in B + +//--- C/Header.h +// Header.h in C + +//--- vfs/base.yaml +{ + 'version': 0, + 'redirecting-with': 'fallthrough', + 'roots': [ + { 'name': 'NAME_DIR', + 'type': 'directory-remap', + 'external-contents': 'EXTERNAL_DIR' + } + ] +} diff --git a/clang/test/VFS/vfsroot-with-overlay.c b/clang/test/VFS/vfsroot-with-overlay.c --- a/clang/test/VFS/vfsroot-with-overlay.c +++ b/clang/test/VFS/vfsroot-with-overlay.c @@ -1,7 +1,7 @@ // RUN: rm -rf %t // RUN: mkdir -p %t // RUN: sed -e "s@TEST_DIR@%{/S:regex_replacement}@g" -e "s@OUT_DIR@%{/t:regex_replacement}@g" %S/Inputs/vfsroot.yaml > %t.yaml -// RUN: sed -e "s@INPUT_DIR@/indirect-vfs-root-files@g" -e "s@OUT_DIR@/overlay-dir@g" %S/Inputs/vfsoverlay.yaml > %t/vfsoverlay.yaml +// RUN: sed -e "s@INPUT_DIR@%{/S:regex_replacement}/Inputs@g" -e "s@OUT_DIR@/overlay-dir@g" %S/Inputs/vfsoverlay.yaml > %t/vfsoverlay.yaml // RUN: %clang_cc1 -Werror -ivfsoverlay %t.yaml -ivfsoverlay /direct-vfs-root-files/vfsoverlay.yaml -I /overlay-dir -fsyntax-only /tests/vfsroot-with-overlay.c #include "not_real.h" diff --git a/lldb/source/Commands/CommandObjectReproducer.cpp b/lldb/source/Commands/CommandObjectReproducer.cpp --- a/lldb/source/Commands/CommandObjectReproducer.cpp +++ b/lldb/source/Commands/CommandObjectReproducer.cpp @@ -409,23 +409,19 @@ switch (m_options.provider) { case eReproducerProviderFiles: { FileSpec vfs_mapping = loader->GetFile(); + std::string overlay_path = vfs_mapping.GetPath(); - // Read the VFS mapping. - ErrorOr> buffer = - vfs::getRealFileSystem()->getBufferForFile(vfs_mapping.GetPath()); - if (!buffer) { - SetError(result, errorCodeToError(buffer.getError())); + Expected> vfs = + vfs::getVFSFromYAMLs(StringRef(overlay_path)); + if (auto err = vfs.takeError()) { + SetError(result, std::move(err)); return false; } - // Initialize a VFS from the given mapping. - IntrusiveRefCntPtr vfs = vfs::getVFSFromYAML( - std::move(buffer.get()), nullptr, vfs_mapping.GetPath()); - // Dump the VFS to a buffer. std::string str; raw_string_ostream os(str); - static_cast(*vfs).dump(os); + (*vfs)->dump(os); os.flush(); // Return the string. diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -1281,7 +1281,7 @@ return OS.str(); } - StringRef getFileName() { return FileName; } + StringRef getFileName() const { return FileName; } Error takeError() { return Error(std::move(Err)); } diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -306,6 +306,12 @@ /// \returns success if \a path has been made absolute, otherwise a /// platform-specific error_code. virtual std::error_code makeAbsolute(SmallVectorImpl &Path) const; + + virtual void dump(raw_ostream &OS) const {} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif }; /// Gets an \p vfs::FileSystem for the 'real' file system, as seen by @@ -337,6 +343,10 @@ /// their addition. FileSystemList FSList; + /// The last CWD successfully set on at least one of the filesystems in + /// \c FSList. + std::string CWD; + public: OverlayFileSystem(IntrusiveRefCntPtr Base); @@ -347,8 +357,14 @@ llvm::ErrorOr> openFileForRead(const Twine &Path) override; directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override; + + /// Gets the last successfully set CWD, which starts as the CWD from \c Base. llvm::ErrorOr getCurrentWorkingDirectory() const override; + + /// Attempts to set the CWD on all contained FS. Only errors if they all + /// fail. std::error_code setCurrentWorkingDirectory(const Twine &Path) override; + std::error_code isLocal(const Twine &Path, bool &Result) override; std::error_code getRealPath(const Twine &Path, SmallVectorImpl &Output) const override; @@ -357,6 +373,8 @@ using const_iterator = FileSystemList::const_reverse_iterator; using reverse_iterator = FileSystemList::iterator; using const_reverse_iterator = FileSystemList::const_iterator; + using range = iterator_range; + using const_range = iterator_range; /// Get an iterator pointing to the most recently added file system. iterator overlays_begin() { return FSList.rbegin(); } @@ -373,6 +391,12 @@ /// Get an iterator pointing one-past the most recently added file system. reverse_iterator overlays_rend() { return FSList.end(); } const_reverse_iterator overlays_rend() const { return FSList.end(); } + + range overlays_range() { return llvm::reverse(FSList); } + + const_range overlays_range() const { return llvm::reverse(FSList); } + + void dump(raw_ostream &OS) const override; }; /// By default, this delegates all calls to the underlying file system. This @@ -520,19 +544,69 @@ SmallVectorImpl &Output) const override; std::error_code isLocal(const Twine &Path, bool &Result) override; std::error_code setCurrentWorkingDirectory(const Twine &Path) override; + + void dump(raw_ostream &OS) const override; }; /// Get a globally unique ID for a virtual file or directory. llvm::sys::fs::UniqueID getNextVirtualUniqueID(); -/// Gets a \p FileSystem for a virtual file system described in YAML -/// format. +/// \see getVFSFromYAMLs +/// +/// Returns nullptr if \p Buffer could not be parsed as a YAML containing +/// overlay mappings. std::unique_ptr getVFSFromYAML(std::unique_ptr Buffer, llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext = nullptr, IntrusiveRefCntPtr ExternalFS = getRealFileSystem()); +/// Convenience function to read each file from \p ExternalFS and pass their +/// buffers along to the next overload. Returns a \c FileError if any file +/// could not be read. +Expected> +getVFSFromYAMLs(ArrayRef YAMLOverlayPaths, + IntrusiveRefCntPtr ExternalFS = getRealFileSystem(), + SourceMgr::DiagHandlerTy DiagHandler = nullptr, + void *DiagContext = nullptr); + +/// Gets a \c FileSystem that runs operations on the virtual filesystems +/// described by each buffer in \p Buffers and then the \p ExternalFS if those +/// all fail (dependent on the options below). The order that each is run is +/// the reverse of the list, ie. the last buffer creates the first used +/// \c FileSystem. +/// +/// Returns a \c FileError if any of the buffers are invalid and an +/// \c OverlayFileSystem with just the \p ExternalFS if no buffers were +/// provided. +/// +/// Two extra root configuration options are handled by this method when +/// parsing the YAML of each filesystem: +/// 'fallthrough': +/// 'redirecting-with': +/// +/// These specify the order each filesystem is added to an \c OverlayFileSystem +/// and whether they're added at all - +/// - 'fallthrough': allow "falling through" to the next filesystem +/// - 'fallback': run opertions on the next filesystem before this one. +/// Most useful when specified on the first overlay, which +/// causes the \p ExternalFS to run before it (ie. "fallback" +/// to using the mapped paths only when original fails) +/// - 'redirect-only': skip running operations on any further filesystems, +/// ie. this is the last used filesystem. As with +/// 'fallback', this is more useful when specified on the +/// first overlay to avoid also checking the \p ExternalFS. +/// +/// \see OverlayFileSystem +/// \see RedirectingFileSystem +Expected> +getVFSFromYAMLs(ArrayRef YAMLOverlays, + IntrusiveRefCntPtr ExternalFS = getRealFileSystem(), + SourceMgr::DiagHandlerTy DiagHandler = nullptr, + void *DiagContext = nullptr); + struct YAMLVFSEntry { template YAMLVFSEntry(T1 &&VPath, T2 &&RPath, bool IsDirectory = false) @@ -571,10 +645,6 @@ /// 'case-sensitive': /// 'use-external-names': /// 'overlay-relative': -/// 'fallthrough': -/// 'redirecting-with': /// /// Virtual directories that list their contents are represented as /// \verbatim @@ -645,20 +715,6 @@ enum EntryKind { EK_Directory, EK_DirectoryRemap, EK_File }; enum NameKind { NK_NotSet, NK_External, NK_Virtual }; - /// The type of redirection to perform. - enum class RedirectKind { - /// Lookup the redirected path first (ie. the one specified in - /// 'external-contents') and if that fails "fallthrough" to a lookup of the - /// originally provided path. - Fallthrough, - /// Lookup the provided path first and if that fails, "fallback" to a - /// lookup of the redirected path. - Fallback, - /// Only lookup the redirected path, do not lookup the originally provided - /// path. - RedirectOnly - }; - /// A single file or directory in the VFS. class Entry { EntryKind Kind; @@ -688,7 +744,7 @@ DirectoryEntry(StringRef Name, Status S) : Entry(EK_Directory, Name), S(std::move(S)) {} - Status getStatus() { return S; } + Status getStatus() const { return S; } void addContent(std::unique_ptr Content) { Contents.push_back(std::move(Content)); @@ -798,12 +854,6 @@ /// but canonicalize in order to perform the correct entry search. std::error_code makeCanonical(SmallVectorImpl &Path) const; - /// Get the File status, or error, from the underlying external file system. - /// This returns the status with the originally requested name, while looking - /// up the entry using the canonical path. - ErrorOr getExternalStatus(const Twine &CanonicalPath, - const Twine &OriginalPath) const; - // In a RedirectingFileSystem, keys can be specified in Posix or Windows // style (or even a mixture of both), so this comparison helper allows // slashes (representing a root) to match backslashes (and vice versa). Note @@ -824,11 +874,6 @@ /// The file system to use for external references. IntrusiveRefCntPtr ExternalFS; - /// If IsRelativeOverlay is set, this represents the directory - /// path that should be prefixed to each 'external-contents' entry - /// when reading from YAML files. - std::string ExternalContentsPrefixDir; - /// @name Configuration /// @{ @@ -837,17 +882,10 @@ /// Currently, case-insensitive matching only works correctly with ASCII. bool CaseSensitive = is_style_posix(sys::path::Style::native); - /// IsRelativeOverlay marks whether a ExternalContentsPrefixDir path must - /// be prefixed in every 'external-contents' when reading from YAML files. - bool IsRelativeOverlay = false; - /// Whether to use to use the value of 'external-contents' for the /// names of files. This global value is overridable on a per-file basis. bool UseExternalNames = true; - /// Determines the lookups to perform, as well as their order. See - /// \c RedirectKind for details. - RedirectKind Redirection = RedirectKind::Fallthrough; /// @} RedirectingFileSystem(IntrusiveRefCntPtr ExternalFS); @@ -860,18 +898,17 @@ llvm::sys::path::const_iterator End, Entry *From) const; - /// Get the status for a path with the provided \c LookupResult. - ErrorOr status(const Twine &CanonicalPath, const Twine &OriginalPath, - const LookupResult &Result); - public: /// Looks up \p Path in \c Roots and returns a LookupResult giving the /// matched entry and, if the entry was a FileEntry or DirectoryRemapEntry, /// the path it redirects to in the external file system. - ErrorOr lookupPath(StringRef Path) const; + ErrorOr lookupPath(StringRef CanonicalPath) const; - /// Parses \p Buffer, which is expected to be in YAML format and - /// returns a virtual file system representing its contents. + /// Parses \p Buffer, which is expected to be in the YAML format described + /// in \c RedirectingFileSystem and returns a virtual file system + /// representing its contents. + /// + /// \see getVFSFromYAMLs static std::unique_ptr create(std::unique_ptr Buffer, SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, @@ -883,6 +920,7 @@ bool UseExternalNames, FileSystem &ExternalFS); ErrorOr status(const Twine &Path) override; + ErrorOr> openFileForRead(const Twine &Path) override; std::error_code getRealPath(const Twine &Path, @@ -902,19 +940,10 @@ StringRef getExternalContentsPrefixDir() const; - /// Sets the redirection kind to \c Fallthrough if true or \c RedirectOnly - /// otherwise. Will removed in the future, use \c setRedirection instead. - void setFallthrough(bool Fallthrough); - - void setRedirection(RedirectingFileSystem::RedirectKind Kind); - std::vector getRoots() const; - void dump(raw_ostream &OS) const; + void dump(raw_ostream &OS) const override; void dumpEntry(raw_ostream &OS, Entry *E, int NumSpaces = 0) const; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const; -#endif }; /// Collect all pairs of entries from the diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -151,6 +151,10 @@ return Status && Status->exists(); } +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void FileSystem::dump() const { dump(dbgs()); } +#endif + #ifndef NDEBUG static bool isTraversalComponent(StringRef Component) { return Component.equals("..") || Component.equals("."); @@ -273,6 +277,8 @@ std::error_code getRealPath(const Twine &Path, SmallVectorImpl &Output) const override; + virtual void dump(raw_ostream &OS) const override; + private: // If this FS has its own working dir, use it to make Path absolute. // The returned twine is safe to use as long as both Storage and Path live. @@ -354,6 +360,15 @@ return llvm::sys::fs::real_path(adjustPath(Path, Storage), Output); } +void RealFileSystem::dump(raw_ostream &OS) const { + OS << "RealFileSystem using "; + if (WD) + OS << "own"; + else + OS << "process"; + OS << " CWD\n"; +} + IntrusiveRefCntPtr vfs::getRealFileSystem() { static IntrusiveRefCntPtr FS(new RealFileSystem(true)); return FS; @@ -398,67 +413,98 @@ //===-----------------------------------------------------------------------===/ OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr BaseFS) { + if (auto NewCWD = BaseFS->getCurrentWorkingDirectory()) + CWD = *NewCWD; FSList.push_back(std::move(BaseFS)); } void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr FS) { FSList.push_back(FS); - // Synchronize added file systems by duplicating the working directory from - // the first one in the list. - FS->setCurrentWorkingDirectory(getCurrentWorkingDirectory().get()); } ErrorOr OverlayFileSystem::status(const Twine &Path) { // FIXME: handle symlinks that cross file systems - for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { - ErrorOr Status = (*I)->status(Path); - if (Status || Status.getError() != llvm::errc::no_such_file_or_directory) + for (auto &FS : overlays_range()) { + ErrorOr Status = FS->status(Path); + if (Status || Status.getError() != errc::no_such_file_or_directory) return Status; } - return make_error_code(llvm::errc::no_such_file_or_directory); + return errc::no_such_file_or_directory; } ErrorOr> OverlayFileSystem::openFileForRead(const llvm::Twine &Path) { // FIXME: handle symlinks that cross file systems - for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { - auto Result = (*I)->openFileForRead(Path); - if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) + for (auto &FS : overlays_range()) { + auto Result = FS->openFileForRead(Path); + if (Result || Result.getError() != errc::no_such_file_or_directory) return Result; } - return make_error_code(llvm::errc::no_such_file_or_directory); + return errc::no_such_file_or_directory; } -llvm::ErrorOr -OverlayFileSystem::getCurrentWorkingDirectory() const { - // All file systems are synchronized, just take the first working directory. - return FSList.front()->getCurrentWorkingDirectory(); +ErrorOr OverlayFileSystem::getCurrentWorkingDirectory() const { + if (CWD.empty()) + return errc::operation_not_permitted; + return CWD; } std::error_code OverlayFileSystem::setCurrentWorkingDirectory(const Twine &Path) { - for (auto &FS : FSList) - if (std::error_code EC = FS->setCurrentWorkingDirectory(Path)) - return EC; + // There is no guarantee that setting the CWD on the underlying filesystems + // actually succeeds for them all, eg. RealFileSystem only sets its CWD if + // the directory actually exists. But we also don't want to fail if *any* + // fails - it would be a fairly common case that eg. there's a virtual + // directory that doesn't exist in the underlying filesystem. + // + // Instead, keep track of the most-recent CWD that was successfully set on at + // least one FS and use that as the CWD of this FS. + + bool Success = false; + for (auto &FS : overlays_range()) { + std::error_code EC = FS->setCurrentWorkingDirectory(Path); + if (!EC) { + Success = true; + CWD = *FS->getCurrentWorkingDirectory(); + } + } + + // Theoretically all the filesystems could have failed with an unrelated + // error, but this seems the most reasonable error to return. + if (!Success) + return errc::no_such_file_or_directory; return {}; } std::error_code OverlayFileSystem::isLocal(const Twine &Path, bool &Result) { - for (auto &FS : FSList) - if (FS->exists(Path)) - return FS->isLocal(Path, Result); + for (auto &FS : overlays_range()) { + std::error_code EC = FS->isLocal(Path, Result); + if (!EC || EC != errc::no_such_file_or_directory) + return EC; + } return errc::no_such_file_or_directory; } std::error_code OverlayFileSystem::getRealPath(const Twine &Path, SmallVectorImpl &Output) const { - for (const auto &FS : FSList) - if (FS->exists(Path)) - return FS->getRealPath(Path, Output); + for (auto &FS : overlays_range()) { + std::error_code EC = FS->getRealPath(Path, Output); + if (!EC || EC != errc::no_such_file_or_directory) + return EC; + } return errc::no_such_file_or_directory; } +void OverlayFileSystem::dump(raw_ostream &OS) const { + OS << "OverlayFileSystem running operations on: \n"; + for (auto FS : enumerate(overlays_range())) { + OS << FS.index() << ") "; + FS.value()->dump(OS); + OS << "\n"; + } +} + llvm::vfs::detail::DirIterImpl::~DirIterImpl() = default; namespace { @@ -1047,6 +1093,10 @@ return {}; } +void InMemoryFileSystem::dump(raw_ostream &OS) const { + OS << "InMemoryFileSystem\n"; +} + } // namespace vfs } // namespace llvm @@ -1054,8 +1104,6 @@ // RedirectingFileSystem implementation //===-----------------------------------------------------------------------===/ -namespace { - static llvm::sys::path::Style getExistingStyle(llvm::StringRef Path) { // Detect the path style in use by checking the first separator. llvm::sys::path::Style style = llvm::sys::path::Style::native; @@ -1080,17 +1128,6 @@ return result; } -/// Whether the error and entry specify a file/directory that was not found. -static bool isFileNotFound(std::error_code EC, - RedirectingFileSystem::Entry *E = nullptr) { - if (E && !isa(E)) - return false; - return EC == llvm::errc::no_such_file_or_directory; -} - -} // anonymous namespace - - RedirectingFileSystem::RedirectingFileSystem(IntrusiveRefCntPtr FS) : ExternalFS(std::move(FS)) { if (ExternalFS) @@ -1193,14 +1230,15 @@ std::error_code RedirectingFileSystem::setCurrentWorkingDirectory(const Twine &Path) { - // Don't change the working directory if the path doesn't exist. - if (!exists(Path)) - return errc::no_such_file_or_directory; - SmallString<128> AbsolutePath; Path.toVector(AbsolutePath); if (std::error_code EC = makeAbsolute(AbsolutePath)) return EC; + + // Don't change the working directory if the path doesn't exist. + if (!exists(AbsolutePath)) + return errc::no_such_file_or_directory; + WorkingDirectory = std::string(AbsolutePath.str()); return {}; } @@ -1211,7 +1249,7 @@ Path_.toVector(Path); if (std::error_code EC = makeCanonical(Path)) - return {}; + return EC; return ExternalFS->isLocal(Path, Result); } @@ -1255,123 +1293,51 @@ directory_iterator RedirectingFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { - SmallString<256> Path; - Dir.toVector(Path); - - EC = makeCanonical(Path); + SmallString<256> CanonicalPath; + Dir.toVector(CanonicalPath); + EC = makeCanonical(CanonicalPath); if (EC) return {}; - ErrorOr Result = lookupPath(Path); + ErrorOr Result = + lookupPath(CanonicalPath); if (!Result) { - if (Redirection != RedirectKind::RedirectOnly && - isFileNotFound(Result.getError())) - return ExternalFS->dir_begin(Path, EC); - EC = Result.getError(); return {}; } - // Use status to make sure the path exists and refers to a directory. - ErrorOr S = status(Path, Dir, *Result); - if (!S) { - if (Redirection != RedirectKind::RedirectOnly && - isFileNotFound(S.getError(), Result->E)) - return ExternalFS->dir_begin(Dir, EC); + // Create the appropriate directory iterator based on whether we found a + // DirectoryRemapEntry or DirectoryEntry. + Optional ExtRedirect = Result->getExternalRedirect(); + if (!ExtRedirect) { + // DirectoryEntry - just make an iterator for it + auto DE = cast(Result->E); + return directory_iterator(std::make_shared( + CanonicalPath, DE->contents_begin(), DE->contents_end(), EC)); + } + // Have a DirectoryRemapEntry - need to check that the external directory + // actually exists and then possibly remap names to the virtual path depending + // on 'use-external-names'. + ErrorOr S = ExternalFS->status(*ExtRedirect); + if (!S) { EC = S.getError(); return {}; } - if (!S->isDirectory()) { EC = errc::not_a_directory; return {}; } - // Create the appropriate directory iterator based on whether we found a - // DirectoryRemapEntry or DirectoryEntry. - directory_iterator RedirectIter; - std::error_code RedirectEC; - if (auto ExtRedirect = Result->getExternalRedirect()) { - auto RE = cast(Result->E); - RedirectIter = ExternalFS->dir_begin(*ExtRedirect, RedirectEC); - - if (!RE->useExternalName(UseExternalNames)) { - // Update the paths in the results to use the virtual directory's path. - RedirectIter = - directory_iterator(std::make_shared( - std::string(Path), RedirectIter)); - } - } else { - auto DE = cast(Result->E); + auto RE = cast(Result->E); + directory_iterator RedirectIter = ExternalFS->dir_begin(*ExtRedirect, EC); + if (!RE->useExternalName(UseExternalNames)) { + // Update the paths in the results to use the virtual directory's path. RedirectIter = - directory_iterator(std::make_shared( - Path, DE->contents_begin(), DE->contents_end(), RedirectEC)); - } - - if (RedirectEC) { - if (RedirectEC != errc::no_such_file_or_directory) { - EC = RedirectEC; - return {}; - } - RedirectIter = {}; - } - - if (Redirection == RedirectKind::RedirectOnly) { - EC = RedirectEC; - return RedirectIter; + directory_iterator(std::make_shared( + std::string(CanonicalPath), RedirectIter)); } - - std::error_code ExternalEC; - directory_iterator ExternalIter = ExternalFS->dir_begin(Path, ExternalEC); - if (ExternalEC) { - if (ExternalEC != errc::no_such_file_or_directory) { - EC = ExternalEC; - return {}; - } - ExternalIter = {}; - } - - SmallVector Iters; - switch (Redirection) { - case RedirectKind::Fallthrough: - Iters.push_back(ExternalIter); - Iters.push_back(RedirectIter); - break; - case RedirectKind::Fallback: - Iters.push_back(RedirectIter); - Iters.push_back(ExternalIter); - break; - default: - llvm_unreachable("unhandled RedirectKind"); - } - - directory_iterator Combined{ - std::make_shared(Iters, EC)}; - if (EC) - return {}; - return Combined; -} - -void RedirectingFileSystem::setExternalContentsPrefixDir(StringRef PrefixDir) { - ExternalContentsPrefixDir = PrefixDir.str(); -} - -StringRef RedirectingFileSystem::getExternalContentsPrefixDir() const { - return ExternalContentsPrefixDir; -} - -void RedirectingFileSystem::setFallthrough(bool Fallthrough) { - if (Fallthrough) { - Redirection = RedirectingFileSystem::RedirectKind::Fallthrough; - } else { - Redirection = RedirectingFileSystem::RedirectKind::RedirectOnly; - } -} - -void RedirectingFileSystem::setRedirection( - RedirectingFileSystem::RedirectKind Kind) { - Redirection = Kind; + return RedirectIter; } std::vector RedirectingFileSystem::getRoots() const { @@ -1382,6 +1348,7 @@ } void RedirectingFileSystem::dump(raw_ostream &OS) const { + OS << "RedirectingFileSystem\n"; for (const auto &Root : Roots) dumpEntry(OS, Root.get()); } @@ -1392,22 +1359,50 @@ StringRef Name = E->getName(); for (int i = 0, e = NumSpaces; i < e; ++i) OS << " "; - OS << "'" << Name.str().c_str() << "'" - << "\n"; + OS << "'" << Name.str().c_str() << "'"; - if (E->getKind() == RedirectingFileSystem::EK_Directory) { - auto *DE = dyn_cast(E); - assert(DE && "Should be a directory"); + switch (E->getKind()) { + case EK_Directory: { + auto *DE = cast(E); + OS << "\n"; for (std::unique_ptr &SubEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) dumpEntry(OS, SubEntry.get(), NumSpaces + 2); + break; + } + case EK_DirectoryRemap: { + auto *DRE = cast(E); + OS << " -> '" << DRE->getExternalContentsPath() << "'\n"; + break; + } + case EK_File: + OS << "\n"; + break; } } -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void RedirectingFileSystem::dump() const { dump(dbgs()); } -#endif +namespace { + +struct YAMLParseResult { + /// The way in which this VFS should be added to an overlay. Note that the + /// order is right to left - given [A, B, C], any FS operations run on C + /// *first* and if failing, B, followed by A. + enum class RedirectKind { + /// Allow "falling through" to the next (ie. to the left) VFS. + Fallthrough, + /// Run any operations on the next VFS *first*, using this VFS as a + /// "fallback". + Fallback, + /// Do not perform any more operations on further VFS. + RedirectOnly + }; + + std::unique_ptr FS; + RedirectKind Redirection; +}; + +} // namespace /// A helper class to hold the common YAML parsing state. class llvm::vfs::RedirectingFileSystemParser { @@ -1450,19 +1445,18 @@ return false; } - Optional - parseRedirectKind(yaml::Node *N) { + Optional parseRedirectKind(yaml::Node *N) { SmallString<12> Storage; StringRef Value; if (!parseScalarString(N, Value, Storage)) return None; if (Value.equals_insensitive("fallthrough")) { - return RedirectingFileSystem::RedirectKind::Fallthrough; + return YAMLParseResult::RedirectKind::Fallthrough; } else if (Value.equals_insensitive("fallback")) { - return RedirectingFileSystem::RedirectKind::Fallback; + return YAMLParseResult::RedirectKind::Fallback; } else if (Value.equals_insensitive("redirect-only")) { - return RedirectingFileSystem::RedirectKind::RedirectOnly; + return YAMLParseResult::RedirectKind::RedirectOnly; } return None; } @@ -1505,10 +1499,10 @@ public: static RedirectingFileSystem::Entry * - lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name, + lookupOrCreateEntry(RedirectingFileSystem &FS, StringRef Name, RedirectingFileSystem::Entry *ParentEntry = nullptr) { if (!ParentEntry) { // Look for a existent root - for (const auto &Root : FS->Roots) { + for (const auto &Root : FS.Roots) { if (Name.equals(Root->getName())) { ParentEntry = Root.get(); return ParentEntry; @@ -1533,8 +1527,8 @@ file_type::directory_file, sys::fs::all_all)); if (!ParentEntry) { // Add a new root to the overlay - FS->Roots.push_back(std::move(E)); - ParentEntry = FS->Roots.back().get(); + FS.Roots.push_back(std::move(E)); + ParentEntry = FS.Roots.back().get(); return ParentEntry; } @@ -1544,7 +1538,7 @@ } private: - void uniqueOverlayTree(RedirectingFileSystem *FS, + void uniqueOverlayTree(RedirectingFileSystem &FS, RedirectingFileSystem::Entry *SrcE, RedirectingFileSystem::Entry *NewParentE = nullptr) { StringRef Name = SrcE->getName(); @@ -1582,7 +1576,8 @@ } std::unique_ptr - parseEntry(yaml::Node *N, RedirectingFileSystem *FS, bool IsRootEntry) { + parseEntry(yaml::Node *N, bool IsRelativeOverlay, + StringRef ExternalContentsPrefixDir, bool IsRootEntry) { auto *M = dyn_cast(N); if (!M) { error(N, "expected mapping node for file or directory entry"); @@ -1657,7 +1652,8 @@ for (auto &I : *Contents) { if (std::unique_ptr E = - parseEntry(&I, FS, /*IsRootEntry*/ false)) + parseEntry(&I, IsRelativeOverlay, ExternalContentsPrefixDir, + /*IsRootEntry=*/false)) EntryArrayContents.push_back(std::move(E)); else return nullptr; @@ -1673,8 +1669,8 @@ return nullptr; SmallString<256> FullPath; - if (FS->IsRelativeOverlay) { - FullPath = FS->getExternalContentsPrefixDir(); + if (IsRelativeOverlay) { + FullPath.append(ExternalContentsPrefixDir); assert(!FullPath.empty() && "External contents prefix directory must exist"); llvm::sys::path::append(FullPath, Value); @@ -1796,12 +1792,45 @@ public: RedirectingFileSystemParser(yaml::Stream &S) : Stream(S) {} - // false on error - bool parse(yaml::Node *Root, RedirectingFileSystem *FS) { + static Optional + parse(MemoryBufferRef Buffer, SourceMgr &SM, + IntrusiveRefCntPtr ExternalFS) { + yaml::Stream Stream(Buffer, SM); + yaml::document_iterator DI = Stream.begin(); + yaml::Node *Root = DI->getRoot(); + if (DI == Stream.end() || !Root) { + SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node"); + return None; + } + + SmallString<256> ExternalContentsPrefixDir; + if (!Buffer.getBufferIdentifier().empty()) { + // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed + // to each 'external-contents' path. + // + // Example: + // -ivfsoverlay dummy.cache/vfs/vfs.yaml + // yields: + // FS->ExternalContentsPrefixDir => //dummy.cache/vfs + // + ExternalContentsPrefixDir = + sys::path::parent_path(Buffer.getBufferIdentifier()); + std::error_code EC = ExternalFS->makeAbsolute(ExternalContentsPrefixDir); + assert(!EC && "Overlay dir final path must be absolute"); + (void)EC; + } + + RedirectingFileSystemParser P(Stream); + return P.parse(Root, ExternalContentsPrefixDir, std::move(ExternalFS)); + } + + Optional parse(yaml::Node *Root, + StringRef ExternalContentsPrefixDir, + IntrusiveRefCntPtr ExternalFS) { auto *Top = dyn_cast(Root); if (!Top) { error(Root, "expected mapping node"); - return false; + return None; } KeyStatusPair Fields[] = { @@ -1817,85 +1846,91 @@ DenseMap Keys(std::begin(Fields), std::end(Fields)); std::vector> RootEntries; + YAMLParseResult Result{std::unique_ptr( + new RedirectingFileSystem(ExternalFS)), + YAMLParseResult::RedirectKind::Fallthrough}; + bool IsRelativeOverlay = false; + // Parse configuration and 'roots' for (auto &I : *Top) { SmallString<10> KeyBuffer; StringRef Key; if (!parseScalarString(I.getKey(), Key, KeyBuffer)) - return false; + return None; if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys)) - return false; + return None; if (Key == "roots") { auto *Roots = dyn_cast(I.getValue()); if (!Roots) { error(I.getValue(), "expected array"); - return false; + return None; } for (auto &I : *Roots) { if (std::unique_ptr E = - parseEntry(&I, FS, /*IsRootEntry*/ true)) + parseEntry(&I, IsRelativeOverlay, ExternalContentsPrefixDir, + /*IsRootEntry=*/true)) RootEntries.push_back(std::move(E)); else - return false; + return None; } } else if (Key == "version") { StringRef VersionString; SmallString<4> Storage; if (!parseScalarString(I.getValue(), VersionString, Storage)) - return false; + return None; int Version; if (VersionString.getAsInteger(10, Version)) { error(I.getValue(), "expected integer"); - return false; + return None; } if (Version < 0) { error(I.getValue(), "invalid version number"); - return false; + return None; } if (Version != 0) { error(I.getValue(), "version mismatch, expected 0"); - return false; + return None; } } else if (Key == "case-sensitive") { - if (!parseScalarBool(I.getValue(), FS->CaseSensitive)) - return false; + if (!parseScalarBool(I.getValue(), Result.FS->CaseSensitive)) + return None; } else if (Key == "overlay-relative") { - if (!parseScalarBool(I.getValue(), FS->IsRelativeOverlay)) - return false; + if (!parseScalarBool(I.getValue(), IsRelativeOverlay)) + return None; } else if (Key == "use-external-names") { - if (!parseScalarBool(I.getValue(), FS->UseExternalNames)) - return false; + if (!parseScalarBool(I.getValue(), Result.FS->UseExternalNames)) + return None; } else if (Key == "fallthrough") { if (Keys["redirecting-with"].Seen) { error(I.getValue(), "'fallthrough' and 'redirecting-with' are mutually exclusive"); - return false; + return None; } bool ShouldFallthrough = false; if (!parseScalarBool(I.getValue(), ShouldFallthrough)) - return false; + return None; if (ShouldFallthrough) { - FS->Redirection = RedirectingFileSystem::RedirectKind::Fallthrough; + Result.Redirection = YAMLParseResult::RedirectKind::Fallthrough; } else { - FS->Redirection = RedirectingFileSystem::RedirectKind::RedirectOnly; + Result.Redirection = YAMLParseResult::RedirectKind::RedirectOnly; } } else if (Key == "redirecting-with") { if (Keys["fallthrough"].Seen) { error(I.getValue(), "'fallthrough' and 'redirecting-with' are mutually exclusive"); - return false; + return None; } if (auto Kind = parseRedirectKind(I.getValue())) { - FS->Redirection = *Kind; + Result.Redirection = *Kind; } else { error(I.getValue(), "expected valid redirect kind"); - return false; + return None; } } else { llvm_unreachable("key missing from Keys"); @@ -1903,18 +1938,18 @@ } if (Stream.failed()) - return false; + return None; if (!checkMissingKeys(Top, Keys)) - return false; + return None; // Now that we sucessefully parsed the YAML file, canonicalize the internal // representation to a proper directory tree so that we can search faster // inside the VFS. for (auto &E : RootEntries) - uniqueOverlayTree(FS, E.get()); + uniqueOverlayTree(*Result.FS, E.get()); - return true; + return Result; } }; @@ -1924,41 +1959,12 @@ StringRef YAMLFilePath, void *DiagContext, IntrusiveRefCntPtr ExternalFS) { SourceMgr SM; - yaml::Stream Stream(Buffer->getMemBufferRef(), SM); - SM.setDiagHandler(DiagHandler, DiagContext); - yaml::document_iterator DI = Stream.begin(); - yaml::Node *Root = DI->getRoot(); - if (DI == Stream.end() || !Root) { - SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node"); - return nullptr; - } - RedirectingFileSystemParser P(Stream); - - std::unique_ptr FS( - new RedirectingFileSystem(ExternalFS)); - - if (!YAMLFilePath.empty()) { - // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed - // to each 'external-contents' path. - // - // Example: - // -ivfsoverlay dummy.cache/vfs/vfs.yaml - // yields: - // FS->ExternalContentsPrefixDir => //dummy.cache/vfs - // - SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath); - std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir); - assert(!EC && "Overlay dir final path must be absolute"); - (void)EC; - FS->setExternalContentsPrefixDir(OverlayAbsDir); - } - - if (!P.parse(Root, FS.get())) - return nullptr; - - return FS; + if (Optional Result = RedirectingFileSystemParser::parse( + Buffer->getMemBufferRef(), SM, std::move(ExternalFS))) + return std::move(Result->FS); + return nullptr; } std::unique_ptr RedirectingFileSystem::create( @@ -1991,8 +1997,8 @@ for (auto I = llvm::sys::path::begin(FromDirectory), E = llvm::sys::path::end(FromDirectory); I != E; ++I) { - Parent = RedirectingFileSystemParser::lookupOrCreateEntry(FS.get(), *I, - Parent); + Parent = + RedirectingFileSystemParser::lookupOrCreateEntry(*FS, *I, Parent); } assert(Parent && "File without a directory?"); { @@ -2044,9 +2050,9 @@ } ErrorOr -RedirectingFileSystem::lookupPath(StringRef Path) const { - sys::path::const_iterator Start = sys::path::begin(Path); - sys::path::const_iterator End = sys::path::end(Path); +RedirectingFileSystem::lookupPath(StringRef CanonicalPath) const { + sys::path::const_iterator Start = sys::path::begin(CanonicalPath); + sys::path::const_iterator End = sys::path::end(CanonicalPath); for (const auto &Root : Roots) { ErrorOr Result = lookupPathImpl(Start, End, Root.get()); @@ -2097,82 +2103,37 @@ return make_error_code(llvm::errc::no_such_file_or_directory); } -static Status getRedirectedFileStatus(const Twine &OriginalPath, - bool UseExternalNames, - Status ExternalStatus) { - Status S = ExternalStatus; - if (!UseExternalNames) - S = Status::copyWithNewName(S, OriginalPath); - S.IsVFSMapped = true; - return S; -} - -ErrorOr RedirectingFileSystem::status( - const Twine &CanonicalPath, const Twine &OriginalPath, - const RedirectingFileSystem::LookupResult &Result) { - if (Optional ExtRedirect = Result.getExternalRedirect()) { - SmallString<256> CanonicalRemappedPath((*ExtRedirect).str()); - if (std::error_code EC = makeCanonical(CanonicalRemappedPath)) - return EC; - - ErrorOr S = ExternalFS->status(CanonicalRemappedPath); - if (!S) - return S; - S = Status::copyWithNewName(*S, *ExtRedirect); - auto *RE = cast(Result.E); - return getRedirectedFileStatus(OriginalPath, - RE->useExternalName(UseExternalNames), *S); - } - - auto *DE = cast(Result.E); - return Status::copyWithNewName(DE->getStatus(), CanonicalPath); -} - -ErrorOr -RedirectingFileSystem::getExternalStatus(const Twine &CanonicalPath, - const Twine &OriginalPath) const { - if (auto Result = ExternalFS->status(CanonicalPath)) { - return Result.get().copyWithNewName(Result.get(), OriginalPath); - } else { - return Result.getError(); - } -} - ErrorOr RedirectingFileSystem::status(const Twine &OriginalPath) { SmallString<256> CanonicalPath; OriginalPath.toVector(CanonicalPath); - if (std::error_code EC = makeCanonical(CanonicalPath)) return EC; - if (Redirection == RedirectKind::Fallback) { - // Attempt to find the original file first, only falling back to the - // mapped file if that fails. - ErrorOr S = getExternalStatus(CanonicalPath, OriginalPath); - if (S) - return S; - } - ErrorOr Result = lookupPath(CanonicalPath); - if (!Result) { - // Was not able to map file, fallthrough to using the original path if - // that was the specified redirection type. - if (Redirection == RedirectKind::Fallthrough && - isFileNotFound(Result.getError())) - return getExternalStatus(CanonicalPath, OriginalPath); + if (!Result) return Result.getError(); - } - ErrorOr S = status(CanonicalPath, OriginalPath, *Result); - if (!S && Redirection == RedirectKind::Fallthrough && - isFileNotFound(S.getError(), Result->E)) { - // Mapped the file but it wasn't found in the underlying filesystem, - // fallthrough to using the original path if that was the specified - // redirection type. - return getExternalStatus(CanonicalPath, OriginalPath); + Optional ExtRedirect = Result->getExternalRedirect(); + if (!ExtRedirect) { + auto *DE = cast(Result->E); + Status S = DE->getStatus(); + if (!UseExternalNames) + S = Status::copyWithNewName(S, OriginalPath); + else + S = Status::copyWithNewName(S, CanonicalPath); + S.IsVFSMapped = true; + return S; } + ErrorOr S = ExternalFS->status(*ExtRedirect); + if (!S) + return S; + + auto *RE = cast(Result->E); + if (!RE->useExternalName(UseExternalNames)) + S = Status::copyWithNewName(*S, OriginalPath); + S->IsVFSMapped = true; return S; } @@ -2188,8 +2149,8 @@ : InnerFile(std::move(InnerFile)), S(std::move(S)) {} ErrorOr status() override { return S; } - ErrorOr> + ErrorOr> getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) override { return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator, @@ -2219,65 +2180,32 @@ RedirectingFileSystem::openFileForRead(const Twine &OriginalPath) { SmallString<256> CanonicalPath; OriginalPath.toVector(CanonicalPath); - if (std::error_code EC = makeCanonical(CanonicalPath)) return EC; - if (Redirection == RedirectKind::Fallback) { - // Attempt to find the original file first, only falling back to the - // mapped file if that fails. - auto F = File::getWithPath(ExternalFS->openFileForRead(CanonicalPath), - OriginalPath); - if (F) - return F; - } - ErrorOr Result = lookupPath(CanonicalPath); - if (!Result) { - // Was not able to map file, fallthrough to using the original path if - // that was the specified redirection type. - if (Redirection == RedirectKind::Fallthrough && - isFileNotFound(Result.getError())) - return File::getWithPath(ExternalFS->openFileForRead(CanonicalPath), - OriginalPath); + if (!Result) return Result.getError(); - } - if (!Result->getExternalRedirect()) // FIXME: errc::not_a_file? + Optional ExtRedirect = Result->getExternalRedirect(); + if (!ExtRedirect) return make_error_code(llvm::errc::invalid_argument); - StringRef ExtRedirect = *Result->getExternalRedirect(); - SmallString<256> CanonicalRemappedPath(ExtRedirect.str()); - if (std::error_code EC = makeCanonical(CanonicalRemappedPath)) - return EC; - auto *RE = cast(Result->E); - - auto ExternalFile = File::getWithPath( - ExternalFS->openFileForRead(CanonicalRemappedPath), ExtRedirect); - if (!ExternalFile) { - if (Redirection == RedirectKind::Fallthrough && - isFileNotFound(ExternalFile.getError(), Result->E)) { - // Mapped the file but it wasn't found in the underlying filesystem, - // fallthrough to using the original path if that was the specified - // redirection type. - return File::getWithPath(ExternalFS->openFileForRead(CanonicalPath), - OriginalPath); - } + ErrorOr> ExternalFile = + ExternalFS->openFileForRead(*ExtRedirect); + if (!RE->useExternalName(UseExternalNames)) + ExternalFile = File::getWithPath(std::move(ExternalFile), OriginalPath); + if (!ExternalFile) return ExternalFile; - } - auto ExternalStatus = (*ExternalFile)->status(); + ErrorOr ExternalStatus = (*ExternalFile)->status(); if (!ExternalStatus) return ExternalStatus.getError(); - - // Otherwise, the file was successfully remapped. Mark it as such. Also - // replace the underlying path if the external name is being used. - Status S = getRedirectedFileStatus( - OriginalPath, RE->useExternalName(UseExternalNames), *ExternalStatus); - return std::unique_ptr( - std::make_unique(std::move(*ExternalFile), S)); + ExternalStatus->IsVFSMapped = true; + return std::make_unique(std::move(*ExternalFile), + *ExternalStatus); } std::error_code @@ -2285,48 +2213,20 @@ SmallVectorImpl &Output) const { SmallString<256> CanonicalPath; OriginalPath.toVector(CanonicalPath); - if (std::error_code EC = makeCanonical(CanonicalPath)) return EC; - if (Redirection == RedirectKind::Fallback) { - // Attempt to find the original file first, only falling back to the - // mapped file if that fails. - std::error_code EC = ExternalFS->getRealPath(CanonicalPath, Output); - if (!EC) - return EC; - } - ErrorOr Result = lookupPath(CanonicalPath); - if (!Result) { - // Was not able to map file, fallthrough to using the original path if - // that was the specified redirection type. - if (Redirection == RedirectKind::Fallthrough && - isFileNotFound(Result.getError())) - return ExternalFS->getRealPath(CanonicalPath, Output); + if (!Result) return Result.getError(); - } - // If we found FileEntry or DirectoryRemapEntry, look up the mapped - // path in the external file system. - if (auto ExtRedirect = Result->getExternalRedirect()) { - auto P = ExternalFS->getRealPath(*ExtRedirect, Output); - if (P && Redirection == RedirectKind::Fallthrough && - isFileNotFound(P, Result->E)) { - // Mapped the file but it wasn't found in the underlying filesystem, - // fallthrough to using the original path if that was the specified - // redirection type. - return ExternalFS->getRealPath(CanonicalPath, Output); - } - return P; - } + if (Optional ExternalRedirect = Result->getExternalRedirect()) + return ExternalFS->getRealPath(*ExternalRedirect, Output); - // If we found a DirectoryEntry, still fallthrough to the original path if - // allowed, because directories don't have a single external contents path. - if (Redirection == RedirectKind::Fallthrough) - return ExternalFS->getRealPath(CanonicalPath, Output); - return llvm::errc::invalid_argument; + // Otherwise we have a DirectoryEntry, just use its path + Output.assign(CanonicalPath); + return {}; } std::unique_ptr @@ -2334,9 +2234,132 @@ SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext, IntrusiveRefCntPtr ExternalFS) { - return RedirectingFileSystem::create(std::move(Buffer), DiagHandler, - YAMLFilePath, DiagContext, - std::move(ExternalFS)); + MemoryBufferRef BufferRef {Buffer->getBuffer(), YAMLFilePath}; + auto FS = getVFSFromYAMLs(BufferRef, ExternalFS, DiagHandler, DiagContext); + if (auto Err = FS.takeError()) { + consumeError(std::move(Err)); + return nullptr; + } + return std::move(*FS); +} + +namespace { + +struct VFSResult { + IntrusiveRefCntPtr FS; + YAMLParseResult::RedirectKind Redirection; + + VFSResult(IntrusiveRefCntPtr FS, + YAMLParseResult::RedirectKind Redirection) + : FS(std::move(FS)), Redirection(Redirection) {} + + VFSResult(YAMLParseResult &Result) + : FS(std::move(Result.FS)), Redirection(Result.Redirection) {} +}; + +} // namespace + +static std::unique_ptr +createOverlay(ArrayRef VFSResults) { + if (VFSResults.empty()) + return nullptr; + + auto OverlayFS = std::make_unique(VFSResults.front().FS); + for (const auto &Result : VFSResults.drop_front()) { + OverlayFS->pushOverlay(Result.FS); + } + return OverlayFS; +} + +Expected> +vfs::getVFSFromYAMLs(ArrayRef YAMLOverlayPaths, + IntrusiveRefCntPtr ExternalFS, + SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext) { + SmallVector BufferRefs; + for (StringRef Path : YAMLOverlayPaths) { + BufferRefs.emplace_back(StringRef(), Path); + } + return getVFSFromYAMLs(BufferRefs, std::move(ExternalFS), DiagHandler, + DiagContext); +} + +Expected> +vfs::getVFSFromYAMLs(ArrayRef YAMLOverlays, + IntrusiveRefCntPtr ExternalFS, + SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext) { + if (YAMLOverlays.empty()) + return std::make_unique(std::move(ExternalFS)); + + SourceMgr SM; + SM.setDiagHandler(DiagHandler, DiagContext); + + SmallVector Overlays; + Overlays.emplace_back(ExternalFS, YAMLParseResult::RedirectKind::Fallthrough); + std::unique_ptr OverlayFS = createOverlay(Overlays); + + for (MemoryBufferRef Buffer : YAMLOverlays) { + // Bit of a hack. If the buffer's data is a nullptr (not just empty), + // attempt to read the the file from the current FS. This is to + // differentiate between an actual buffer and one that was made from just + // a path. The paths cannot be read all up front in order to support + // overlay files being defined in a VFS. + std::unique_ptr TempBuffer; + if (Buffer.getBuffer().data() == nullptr) { + ErrorOr> NewBufferOrError = + OverlayFS->getBufferForFile(Buffer.getBufferIdentifier()); + if (!NewBufferOrError) + return createFileError(Buffer.getBufferIdentifier(), + NewBufferOrError.getError()); + TempBuffer = std::move(*NewBufferOrError); + Buffer = MemoryBufferRef(TempBuffer->getBuffer(), + Buffer.getBufferIdentifier()); + } + + Optional Result = + RedirectingFileSystemParser::parse(Buffer, SM, ExternalFS); + if (!Result) + return createFileError(Buffer.getBufferIdentifier(), + llvm::errc::invalid_argument); + + switch (Result->Redirection) { + case YAMLParseResult::RedirectKind::Fallthrough: + // Simple case - add on the FS + Overlays.emplace_back(*Result); + OverlayFS->pushOverlay(Overlays.back().FS); + break; + case YAMLParseResult::RedirectKind::Fallback: + // Fallback implies that the previously added FS should run operations + // before hitting this FS. This doesn't make a whole lot of sense for any + // position other than the beginning where we want the external FS to run + // first. For other positions the overlays could just be specified in the + // opposite order. + // + // This is fallout from the history of \c RedirectingFileSystem. Ideally + // we would just depend on order and have some way to specify the real + // filesystem. + + if (Overlays.back().Redirection != + YAMLParseResult::RedirectKind::RedirectOnly) { + // If the previously inserted overlay is redirect only, don't bother + // adding this one - it would never run anyway (since redirect only + // is necessarily the last FS). + + // Note: Overlays is never empty, it always has at least the + // \c ExternalFS + Overlays.insert(Overlays.end() - 1, VFSResult(*Result)); + OverlayFS = createOverlay(Overlays); + } + break; + case YAMLParseResult::RedirectKind::RedirectOnly: + // This is now the last FS, clear the rest if there were any + Overlays.clear(); + Overlays.emplace_back(*Result); + OverlayFS = createOverlay(Overlays); + break; + } + } + + return OverlayFS; } static void getVFSEntries(RedirectingFileSystem::Entry *SrcE, diff --git a/llvm/tools/dsymutil/Reproducer.h b/llvm/tools/dsymutil/Reproducer.h --- a/llvm/tools/dsymutil/Reproducer.h +++ b/llvm/tools/dsymutil/Reproducer.h @@ -59,8 +59,7 @@ }; /// Reproducer instance used to use an existing reproducer. The VFS returned by -/// this instance is a RedirectingFileSystem that remaps paths to their -/// counterpart in the reproducer. +/// this instance remaps paths to their counterpart in the reproducer. class ReproducerUse : public Reproducer { public: ReproducerUse(StringRef Root, std::error_code &EC); diff --git a/llvm/tools/dsymutil/Reproducer.cpp b/llvm/tools/dsymutil/Reproducer.cpp --- a/llvm/tools/dsymutil/Reproducer.cpp +++ b/llvm/tools/dsymutil/Reproducer.cpp @@ -48,15 +48,15 @@ ReproducerUse::ReproducerUse(StringRef Root, std::error_code &EC) { SmallString<128> Mapping(Root); sys::path::append(Mapping, "mapping.yaml"); - ErrorOr> Buffer = - vfs::getRealFileSystem()->getBufferForFile(Mapping.str()); - if (!Buffer) { - EC = Buffer.getError(); + auto OverlayFS = llvm::vfs::getVFSFromYAMLs(Mapping.str()); + if (auto Err = OverlayFS.takeError()) { + llvm::handleAllErrors(std::move(Err), [&](const llvm::ErrorInfoBase &E) { + EC = E.convertToErrorCode(); + }); return; } - - VFS = llvm::vfs::getVFSFromYAML(std::move(Buffer.get()), nullptr, Mapping); + VFS = std::move(*OverlayFS); } llvm::Expected> diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp --- a/llvm/unittests/Support/VirtualFileSystemTest.cpp +++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp @@ -14,6 +14,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Testing/Support/Error.h" #include "llvm/Testing/Support/SupportHelpers.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -40,6 +41,7 @@ llvm_unreachable("unimplemented"); } std::error_code close() override { return std::error_code(); } + void setPath(const Twine &Path) override { S = S.copyWithNewName(S, Path); } }; class DummyFileSystem : public vfs::FileSystem { @@ -171,6 +173,8 @@ sys::fs::file_type::symlink_file, sys::fs::all_all); addEntry(Path, S); } + + void dump(raw_ostream &OS) const override { OS << "DummyFileSystem\n"; } }; class ErrorDummyFileSystem : public DummyFileSystem { @@ -848,6 +852,73 @@ } } +TEST(OverlayFileSystemTest, BaseNoWorkingDir) { + auto ErrorFS = makeIntrusiveRefCnt(); + // Base has no CWD, Overlay shouldn't either + auto OverlayFS = makeIntrusiveRefCnt(ErrorFS); + ASSERT_TRUE(OverlayFS->getCurrentWorkingDirectory().getError()); + + auto DummyFS = makeIntrusiveRefCnt(); + DummyFS->setCurrentWorkingDirectory("/root"); + + // Should still have CWD on underlying + OverlayFS->pushOverlay(DummyFS); + auto CWD = DummyFS->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root", *CWD); + // But not on overlay + CWD = OverlayFS->getCurrentWorkingDirectory(); + ASSERT_TRUE(CWD.getError()); + + // setCWD should now succeed + auto EC = OverlayFS->setCurrentWorkingDirectory("/root2"); + ASSERT_FALSE(EC); + CWD = OverlayFS->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root2", *CWD); + + // And change the underlying as well + CWD = DummyFS->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root2", *CWD); +} + +TEST(OverlayFileSystemTest, BaseWithWorkingDir) { + // Base has CWD, overlay should too + auto DummyFS1 = makeIntrusiveRefCnt(); + DummyFS1->setCurrentWorkingDirectory("/root"); + auto OverlayFS = makeIntrusiveRefCnt(DummyFS1); + auto CWD = OverlayFS->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root", *CWD); + + // Check CWD is uneffected when adding to an overlay + auto DummyFS2 = makeIntrusiveRefCnt(); + DummyFS2->setCurrentWorkingDirectory("/root2"); + OverlayFS->pushOverlay(DummyFS2); + CWD = DummyFS2->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root2", *CWD); + + // setCWD should still work, even if a FS fails + auto ErrorFS = makeIntrusiveRefCnt(); + OverlayFS->pushOverlay(ErrorFS); + auto EC = OverlayFS->setCurrentWorkingDirectory("/root3"); + ASSERT_FALSE(EC); + + CWD = OverlayFS->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root3", *CWD); + + // And it should set all underlying filesystems + CWD = DummyFS1->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root3", *CWD); + CWD = DummyFS2->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + ASSERT_EQ("/root3", *CWD); +} + TEST(ProxyFileSystemTest, Basic) { IntrusiveRefCntPtr Base( new vfs::InMemoryFileSystem()); @@ -1629,30 +1700,21 @@ EXPECT_EQ(0, NumDiagnostics); } -TEST_F(VFSFromYAMLTest, ReturnsRequestedPathVFSMiss) { +TEST_F(VFSFromYAMLTest, RelativeVFSPathMiss) { IntrusiveRefCntPtr BaseFS( new vfs::InMemoryFileSystem); BaseFS->addFile("//root/foo/a", 0, MemoryBuffer::getMemBuffer("contents of a")); ASSERT_FALSE(BaseFS->setCurrentWorkingDirectory("//root/foo")); + auto RemappedFS = vfs::RedirectingFileSystem::create( {}, /*UseExternalNames=*/false, *BaseFS); - auto OpenedF = RemappedFS->openFileForRead("a"); + auto OpenedF = BaseFS->openFileForRead("a"); ASSERT_FALSE(OpenedF.getError()); - llvm::ErrorOr Name = (*OpenedF)->getName(); - ASSERT_FALSE(Name.getError()); - EXPECT_EQ("a", Name.get()); - - auto OpenedS = (*OpenedF)->status(); - ASSERT_FALSE(OpenedS.getError()); - EXPECT_EQ("a", OpenedS->getName()); - EXPECT_FALSE(OpenedS->IsVFSMapped); - auto DirectS = RemappedFS->status("a"); - ASSERT_FALSE(DirectS.getError()); - EXPECT_EQ("a", DirectS->getName()); - EXPECT_FALSE(DirectS->IsVFSMapped); + OpenedF = RemappedFS->openFileForRead("a"); + ASSERT_TRUE(OpenedF.getError()); EXPECT_EQ(0, NumDiagnostics); } @@ -2369,7 +2431,7 @@ } TEST_F(VFSFromYAMLTest, WorkingDirectory) { - IntrusiveRefCntPtr Lower(new DummyFileSystem()); + IntrusiveRefCntPtr Lower(new ErrorDummyFileSystem()); Lower->addDirectory("//root/"); Lower->addDirectory("//root/foo"); Lower->addRegularFile("//root/foo/a"); @@ -2391,6 +2453,7 @@ "}", Lower); ASSERT_NE(FS.get(), nullptr); + std::error_code EC = FS->setCurrentWorkingDirectory("//root/bar"); ASSERT_FALSE(EC); @@ -2419,6 +2482,14 @@ ASSERT_TRUE(WorkingDir); EXPECT_EQ(*WorkingDir, "//root/"); + Status = FS->status("bar/a"); + ASSERT_FALSE(Status.getError()); + EXPECT_TRUE(Status->exists()); + + // CWD wasn't set on Lower (it just errors instead), so this should fail + Status = FS->status("foo/a"); + ASSERT_TRUE(Status.getError()); + EC = FS->setCurrentWorkingDirectory("bar"); ASSERT_FALSE(EC); WorkingDir = FS->getCurrentWorkingDirectory(); @@ -2508,43 +2579,6 @@ EXPECT_TRUE(Status->exists()); } -TEST_F(VFSFromYAMLTest, WorkingDirectoryFallthroughInvalid) { - IntrusiveRefCntPtr Lower(new ErrorDummyFileSystem()); - Lower->addDirectory("//root/"); - Lower->addDirectory("//root/foo"); - Lower->addRegularFile("//root/foo/a"); - Lower->addRegularFile("//root/foo/b"); - Lower->addRegularFile("//root/c"); - IntrusiveRefCntPtr FS = getFromYAMLString( - "{ 'use-external-names': false,\n" - " 'roots': [\n" - "{\n" - " 'type': 'directory',\n" - " 'name': '//root/bar',\n" - " 'contents': [ {\n" - " 'type': 'file',\n" - " 'name': 'a',\n" - " 'external-contents': '//root/foo/a'\n" - " }\n" - " ]\n" - "}\n" - "]\n" - "}", - Lower); - ASSERT_NE(FS.get(), nullptr); - std::error_code EC = FS->setCurrentWorkingDirectory("//root/"); - ASSERT_FALSE(EC); - ASSERT_NE(FS.get(), nullptr); - - llvm::ErrorOr Status = FS->status("bar/a"); - ASSERT_FALSE(Status.getError()); - EXPECT_TRUE(Status->exists()); - - Status = FS->status("foo/a"); - ASSERT_FALSE(Status.getError()); - EXPECT_TRUE(Status->exists()); -} - TEST_F(VFSFromYAMLTest, VirtualWorkingDirectory) { IntrusiveRefCntPtr Lower(new ErrorDummyFileSystem()); Lower->addDirectory("//root/"); @@ -2928,3 +2962,230 @@ EXPECT_EQ("contents of c", (*BufferKeepA)->getBuffer()); EXPECT_EQ("contents of c", (*BufferExternalA)->getBuffer()); } + +// Check that a mapped directory and its parents all return a status with the +// correct path +TEST(RedirectingFileSystemTest, DirectoryStatus) { + auto Real = makeIntrusiveRefCnt(); + Real->addDirectory("/real"); + + SmallVector> Mappings = { + {"/a/b/c", "/real"}}; + auto FS = vfs::RedirectingFileSystem::create(Mappings, true, *Real); + ASSERT_TRUE(FS); + + auto S = FS->status("/a"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a", S->getName()); + + S = FS->status("/a/b"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/b", S->getName()); + + S = FS->status("/a/b/c"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real", S->getName()); + + FS->setCurrentWorkingDirectory("/a"); + + S = FS->status("b"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/b", S->getName()); + + S = FS->status("b/c"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real", S->getName()); + + FS = vfs::RedirectingFileSystem::create(Mappings, false, *Real); + ASSERT_TRUE(FS); + + // External names is false, use the passed in path instead + FS->setCurrentWorkingDirectory("/a"); + + S = FS->status("b"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("b", S->getName()); + + S = FS->status("b/c"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("b/c", S->getName()); +} + +static std::unique_ptr +getVFSOrNull(ArrayRef YAMLOverlays, + IntrusiveRefCntPtr ExternalFS) { + SmallVector OverlayRefs; + for (const auto &Overlay : YAMLOverlays) { + OverlayRefs.emplace_back(Overlay, ""); + } + + auto ExpectedFS = vfs::getVFSFromYAMLs(OverlayRefs, ExternalFS); + if (auto Err = ExpectedFS.takeError()) { + consumeError(std::move(Err)); + return nullptr; + } + return std::move(*ExpectedFS); +} + +static std::string createSimpleOverlay(StringRef RedirectKind, StringRef From, + StringRef To) { + return ("{\n" + " 'version': 0,\n" + " 'redirecting-with': '" + + RedirectKind + + "'\n" + " 'roots': [\n" + " {\n" + " 'type': 'directory-remap',\n" + " 'name': '" + + From + + "',\n" + " 'external-contents': '" + + To + + "',\n" + " }]\n" + " }" + " ]") + .str(); +} + +// Make sure that overlays are not transitive. Given A -> B and B -> C, if a +// file in A is requested, it should not end up mapping to C. +TEST(VFSFromYAMLsTest, NotTransitive) { + auto C = makeIntrusiveRefCnt(); + C->addDirectory("/real/c"); + C->addRegularFile("/real/c/f"); + + SmallVector Overlays; + Overlays.push_back(createSimpleOverlay("fallthrough", "/b", "/real/c")); + Overlays.push_back(createSimpleOverlay("fallthrough", "/a", "/b")); + auto FS = getVFSOrNull(Overlays, C); + ASSERT_TRUE(FS); + + auto S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real/c/f", S->getName()); + + S = FS->status("/a/f"); + ASSERT_TRUE(S.getError()); +} + +// When fallback is the first overlay, the external FS should be checked before +// it. Given B -> A, A should only be used if the file does not exist in B. +TEST(VFSFromYAMLsTest, FallbackChecksExternalFirst) { + auto AOnly = makeIntrusiveRefCnt(); + AOnly->addDirectory("/a"); + AOnly->addRegularFile("/a/f"); + + auto BOnly = makeIntrusiveRefCnt(); + BOnly->addDirectory("/b"); + BOnly->addRegularFile("/b/f"); + + auto Both = makeIntrusiveRefCnt(); + Both->addDirectory("/a"); + Both->addRegularFile("/a/f"); + Both->addDirectory("/b"); + Both->addRegularFile("/b/f"); + + auto FallbackOverlay = createSimpleOverlay("fallback", "/b", "/a"); + + auto FS = getVFSOrNull(FallbackOverlay, AOnly); + ASSERT_TRUE(FS); + auto S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/f", S->getName()); + + FS = getVFSOrNull(FallbackOverlay, BOnly); + ASSERT_TRUE(FS); + S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/b/f", S->getName()); + + FS = getVFSOrNull(FallbackOverlay, Both); + ASSERT_TRUE(FS); + S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/b/f", S->getName()); +} + +// Ensure the last overlay with redirect-only specified is the final FS +TEST(VFSFromYAMLsTest, RedirectOnlyIsFinalFS) { + auto Real = makeIntrusiveRefCnt(); + Real->addDirectory("/real"); + Real->addRegularFile("/real/f"); + + SmallVector Overlays; + Overlays.push_back(createSimpleOverlay("redirect-only", "/ro1", "/real")); + Overlays.push_back(createSimpleOverlay("fallthrough", "/ft1", "/real")); + Overlays.push_back(createSimpleOverlay("redirect-only", "/ro2", "/real")); + Overlays.push_back(createSimpleOverlay("fallback", "/fb", "/real")); + Overlays.push_back(createSimpleOverlay("fallthrough", "/ft2", "/real")); + auto FS = getVFSOrNull(Overlays, Real); + ASSERT_TRUE(FS); + + // Only //ft2/f and //ro2/f should be valid: + // - //ro1 and //ft1 are specified before //ro2 so never run + // - //fb is specified after, but it's set to fallback and hence //ro2 + // should run first (and not continue) + + auto S = FS->status("/ro1/f"); + ASSERT_TRUE(S.getError()); + + S = FS->status("/ft1/f"); + ASSERT_TRUE(S.getError()); + + S = FS->status("/fb/f"); + ASSERT_TRUE(S.getError()); + + S = FS->status("/ro2/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real/f", S->getName()); + + S = FS->status("/ft2/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real/f", S->getName()); +} + +// Ensure overlays are read from the OverlayFS built so far +TEST(VFSFromYAMLsTest, OverlayFromVFS) { + std::string FT1 = createSimpleOverlay("fallthrough", "/vfs1", "/a"); + std::string FT2 = createSimpleOverlay("fallthrough", "/vfs2", "/b"); + std::string FT3 = createSimpleOverlay("fallthrough", "/vfs3", "/c"); + std::string RO = createSimpleOverlay("redirect-only", "/vfs", "/a"); + + auto Real = makeIntrusiveRefCnt(); + Real->addFile("/ft1.yaml", 0, MemoryBuffer::getMemBuffer(FT1)); + Real->addFile("/ft2.yaml", 0, MemoryBuffer::getMemBuffer(FT2)); + Real->addFile("/a/ft3.yaml", 0, MemoryBuffer::getMemBuffer(FT3)); + Real->addFile("/ro.yaml", 0, MemoryBuffer::getMemBuffer(RO)); + Real->addFile("/a/f", 0, MemoryBuffer::getMemBuffer("a")); + Real->addFile("/b/f", 0, MemoryBuffer::getMemBuffer("b")); + Real->addFile("/c/f", 0, MemoryBuffer::getMemBuffer("c")); + + // 3 to check we're not just using the last overlay + auto ExpectedFS = + vfs::getVFSFromYAMLs({"/ft1.yaml", "/ft2.yaml", "/vfs1/ft3.yaml"}, Real); + ASSERT_THAT_EXPECTED(ExpectedFS, Succeeded()); + auto FS = std::move(*ExpectedFS); + ASSERT_TRUE(FS); + auto S = FS->status("/vfs1/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/f", S->getName()); + S = FS->status("/vfs2/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/b/f", S->getName()); + S = FS->status("/vfs3/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/c/f", S->getName()); + + // If redirect-only is given, any further overlays *must* be specified by it + ExpectedFS = vfs::getVFSFromYAMLs({"/ro.yaml", "/a/ft3.yaml"}, Real); + ASSERT_THAT_EXPECTED(ExpectedFS, Failed()); + + ExpectedFS = vfs::getVFSFromYAMLs({"/ro.yaml", "/vfs/ft3.yaml"}, Real); + FS = std::move(*ExpectedFS); + ASSERT_TRUE(FS); + S = FS->status("/vfs/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/f", S->getName()); +}