Index: clang/test/VFS/Inputs/vfsoverlay-directory-relative.yaml =================================================================== --- /dev/null +++ clang/test/VFS/Inputs/vfsoverlay-directory-relative.yaml @@ -0,0 +1,11 @@ +{ + 'version': 0, + 'fallthrough': true, + 'overlay-relative': true, + 'roots': [ + { 'name': 'OUT_DIR', + 'type': 'directory', + 'external-contents': 'INPUT_DIR' + } + ] +} Index: clang/test/VFS/Inputs/vfsoverlay-directory.yaml =================================================================== --- /dev/null +++ clang/test/VFS/Inputs/vfsoverlay-directory.yaml @@ -0,0 +1,10 @@ +{ + 'version': 0, + 'fallthrough': true, + 'roots': [ + { 'name': 'OUT_DIR', + 'type': 'directory', + 'external-contents': 'INPUT_DIR' + } + ] +} Index: clang/test/VFS/directory.c =================================================================== --- /dev/null +++ clang/test/VFS/directory.c @@ -0,0 +1,48 @@ +// RUN: rm -rf %t +// RUN: mkdir -p %t/Underlying +// RUN: mkdir -p %t/Overlay +// RUN: mkdir -p %t/Middle +// RUN: echo '// B.h in Underlying' > %t/Underlying/B.h +// RUN: echo '#ifdef NESTED' >> %t/Underlying/B.h +// RUN: echo '#include "C.h"' >> %t/Underlying/B.h +// RUN: echo '#endif' >> %t/Underlying/B.h +// RUN: echo '// C.h in Underlying' > %t/Underlying/C.h +// RUN: echo '// C.h in Middle' > %t/Middle/C.h +// RUN: echo '// C.h in Overlay' > %t/Overlay/C.h + +// 1) Underlying -> Overlay (C.h found, B.h falling back to Underlying) +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}/Overlay@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Underlying@g" %S/Inputs/vfsoverlay-directory.yaml > %t/vfs.yaml +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -fsyntax-only -DNESTED -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s +// RUN: sed -e "s@INPUT_DIR@Overlay@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Underlying@g" %S/Inputs/vfsoverlay-directory-relative.yaml > %t/vfs-relative.yaml +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs-relative.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s + +// DIRECT: {{^}}// B.h in Underlying +// DIRECT: {{^}}// C.h in Overlay + +// 2) Underlying -> Middle -> Overlay (C.h found, B.h falling back to Underlying) +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}/Overlay@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Middle@g" %S/Inputs/vfsoverlay-directory.yaml > %t/vfs.yaml +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}/Middle@g" -e "s@OUT_DIR@%{/t:regex_replacement}/Underlying@g" %S/Inputs/vfsoverlay-directory.yaml > %t/vfs2.yaml +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -DNESTED -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=DIRECT %s + +// Same as direct above + +// 3) Underlying -> Middle -> Overlay (C.h falling back to Middle, B.h falling back to Underlying) +// RUN: rm -f %t/Overlay/C.h +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=FALLBACK %s + +// FALLBACK: {{^}}// B.h in Underlying +// FALLBACK: {{^}}// C.h in Middle + +// 3) Underlying -> Middle -> Overlay (C.h falling back to Underlying, B.h falling back to Underlying) +// RUN: rm -f %t/Middle/C.h +// RUN: %clang_cc1 -Werror -I %t/Underlying -ivfsoverlay %t/vfs.yaml -ivfsoverlay %t/vfs2.yaml -fsyntax-only -E -C %s 2>&1 | FileCheck --check-prefix=FALLBACK2 %s + +// FALLBACK2: {{^}}// B.h in Underlying +// FALLBACK2: {{^}}// C.h in Underlying + +#include "B.h" +#ifndef NESTED +#include "C.h" +#endif Index: llvm/include/llvm/Support/VirtualFileSystem.h =================================================================== --- llvm/include/llvm/Support/VirtualFileSystem.h +++ llvm/include/llvm/Support/VirtualFileSystem.h @@ -516,13 +516,15 @@ bool IsDirectory = false; }; -class VFSFromYamlDirIterImpl; +class RedirectingFSDirIterImpl; class RedirectingFileSystemParser; /// A virtual file system parsed from a YAML file. /// -/// Currently, this class allows creating virtual directories and mapping -/// virtual file paths to existing external files, available in \c ExternalFS. +/// Currently, this class allows creating virtual files and directories. Virtual +/// files map to existing external files in \c ExternalFS, and virtual +/// directories may either map to existing directories in \c ExternalFS or list +/// their contents in the form of other virtual directories and/or files. /// /// The basic structure of the parsed file is: /// \verbatim @@ -541,7 +543,7 @@ /// 'overlay-relative': /// 'fallthrough': /// -/// Virtual directories are represented as +/// Virtual directories that list their contents are represented as /// \verbatim /// { /// 'type': 'directory', @@ -550,7 +552,7 @@ /// } /// \endverbatim /// -/// The default attributes for virtual directories are: +/// The default attributes for such virtual directories are: /// \verbatim /// MTime = now() when created /// Perms = 0777 @@ -559,6 +561,17 @@ /// UniqueID = unspecified unique value /// \endverbatim /// +/// Re-mapped directories are represented as +/// /// \verbatim +/// { +/// 'type': 'directory', +/// 'name': , +/// 'external-contents': +/// } +/// \endverbatim +/// +/// and inherit their attributes from the external directory. +/// /// Re-mapped files are represented as /// \verbatim /// { @@ -569,14 +582,15 @@ /// } /// \endverbatim /// -/// and inherit their attributes from the external contents. +/// and similarly inherit their attributes from their external contents. /// -/// In both cases, the 'name' field may contain multiple path components (e.g. -/// /path/to/file). However, any directory that contains more than one child -/// must be uniquely represented by a directory entry. +/// For both files and directories, the 'name' field may contain multiple path +/// components (e.g. /path/to/file). However, any directory that contains more +/// than one child must be uniquely represented by a directory entry. class RedirectingFileSystem : public vfs::FileSystem { public: - enum EntryKind { EK_Directory, EK_File }; + enum EntryKind { EK_Directory, EK_RemapDirectory, EK_File }; + enum NameKind { NK_NotSet, NK_External, NK_Virtual }; /// A single file or directory in the VFS. class Entry { @@ -591,17 +605,54 @@ EntryKind getKind() const { return Kind; } }; - class RedirectingDirectoryEntry : public Entry { + /// A file or directory in the vfs that is mapped to a file or directory in + /// the external filesystem. + class RemapEntry : public Entry { + std::string ExternalContentsPath; + NameKind UseName; + + protected: + RemapEntry(EntryKind K, StringRef Name, StringRef ExternalContentsPath, + NameKind UseName) + : Entry(K, Name), ExternalContentsPath(ExternalContentsPath), + UseName(UseName) {} + + public: + NameKind getUseName() const { return UseName; } + StringRef getExternalContentsPath() const { return ExternalContentsPath; } + + /// whether to use the external path as the name for this file or directory. + bool useExternalName(bool GlobalUseExternalName) const { + return UseName == NK_NotSet ? GlobalUseExternalName + : (UseName == NK_External); + } + + static bool classof(const Entry *E) { + switch (E->getKind()) { + case EK_RemapDirectory: + LLVM_FALLTHROUGH; + case EK_File: + return true; + case EK_Directory: + return false; + } + } + }; + + /// A directory in the vfs with explicitly specified contents. + class DirectoryEntry : public Entry { std::vector> Contents; Status S; public: - RedirectingDirectoryEntry(StringRef Name, - std::vector> Contents, - Status S) + /// Constructs a directory entry with explicitly specified contents. + DirectoryEntry(StringRef Name, std::vector> Contents, + Status S) : Entry(EK_Directory, Name), Contents(std::move(Contents)), S(std::move(S)) {} - RedirectingDirectoryEntry(StringRef Name, Status S) + + /// Constructs an empty directory entry. + DirectoryEntry(StringRef Name, Status S) : Entry(EK_Directory, Name), S(std::move(S)) {} Status getStatus() { return S; } @@ -620,41 +671,41 @@ static bool classof(const Entry *E) { return E->getKind() == EK_Directory; } }; - class RedirectingFileEntry : public Entry { - public: - enum NameKind { NK_NotSet, NK_External, NK_Virtual }; - - private: - std::string ExternalContentsPath; - NameKind UseName; - + /// A directory in the vfs that maps to a directory in the external file + /// system. + class DirectoryRemapEntry : public RemapEntry { public: - RedirectingFileEntry(StringRef Name, StringRef ExternalContentsPath, - NameKind UseName) - : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath), - UseName(UseName) {} - - StringRef getExternalContentsPath() const { return ExternalContentsPath; } + DirectoryRemapEntry(StringRef Name, StringRef ExternalContentsPath, + NameKind UseName) + : RemapEntry(EK_RemapDirectory, Name, ExternalContentsPath, UseName) {} - /// whether to use the external path as the name for this file. - bool useExternalName(bool GlobalUseExternalName) const { - return UseName == NK_NotSet ? GlobalUseExternalName - : (UseName == NK_External); + static bool classof(const Entry *E) { + return E->getKind() == EK_RemapDirectory; } + }; - NameKind getUseName() const { return UseName; } + /// A file in the vfs that maps to a file in the external file system. + class FileEntry : public RemapEntry { + public: + FileEntry(StringRef Name, StringRef ExternalContentsPath, NameKind UseName) + : RemapEntry(EK_File, Name, ExternalContentsPath, UseName) {} static bool classof(const Entry *E) { return E->getKind() == EK_File; } }; private: - friend class VFSFromYamlDirIterImpl; + friend class RedirectingFSDirIterImpl; friend class RedirectingFileSystemParser; bool shouldUseExternalFS() const { return ExternalFSValidWD && IsFallthrough; } + /// Whether to fall back to the external file system when an operation fails + /// with the given error code on a path associated with the provided entry. + bool shouldFallBackToExternalFS(std::error_code Error, + Entry *SrcEntry = nullptr) const; + // In a RedirectingFileSystem, keys can be specified in Posix or Windows // style (or even a mixture of both), so this comparison helper allows // slashes (representing a root) to match backslashes (and vice versa). Note @@ -714,15 +765,17 @@ /// Looks up the path [Start, End) in \p From, possibly /// recursing into the contents of \p From if it is a directory. ErrorOr lookupPath(llvm::sys::path::const_iterator Start, - llvm::sys::path::const_iterator End, - Entry *From) const; + llvm::sys::path::const_iterator End, Entry *From, + SmallVectorImpl &ExternalRedirect) const; /// Get the status of a given an \c Entry. - ErrorOr status(const Twine &Path, Entry *E); + ErrorOr status(const Twine &Path, Entry *E, + StringRef ExternalRedirect); public: /// Looks up \p Path in \c Roots. - ErrorOr lookupPath(const Twine &Path) const; + ErrorOr lookupPath(const Twine &Path, + SmallVectorImpl &ExternalRedirect) const; /// Parses \p Buffer, which is expected to be in YAML format and /// returns a virtual file system representing its contents. Index: llvm/lib/Support/VirtualFileSystem.cpp =================================================================== --- llvm/lib/Support/VirtualFileSystem.cpp +++ llvm/lib/Support/VirtualFileSystem.cpp @@ -162,6 +162,90 @@ } #endif +namespace { + +/// Combines and deduplicates directory entries across multiple file systems. +class CombiningDirIterImpl : public llvm::vfs::detail::DirIterImpl { + using FileSystemPtr = llvm::IntrusiveRefCntPtr; + + /// File systems to check for entries in. Processed in reverse order. + SmallVector FSList; + /// The directory iterator for the current filesystem. + directory_iterator CurrentDirIter; + /// The path of the directory to iterate the entries of. + std::string DirPath; + /// The set of names already returned as entries. + llvm::StringSet<> SeenNames; + + std::error_code incrementImpl(bool IsFirstTime) { + while (true) { + std::error_code EC = incrementDirIter(IsFirstTime); + if (EC || CurrentDirIter == directory_iterator()) { + CurrentEntry = directory_entry(); + return EC; + } + CurrentEntry = *CurrentDirIter; + // FIXME: Check if name is empty? + StringRef Name = llvm::sys::path::filename(CurrentEntry.path()); + if (SeenNames.insert(Name).second) + return EC; // name not seen before + } + llvm_unreachable("returned above"); + } + + std::error_code incrementDirIter(bool IsFirstTime) { + assert((IsFirstTime || CurrentDirIter != directory_iterator()) && + "incrementing past end"); + std::error_code EC; + if (!IsFirstTime) + CurrentDirIter.increment(EC); + if (!EC && CurrentDirIter == directory_iterator()) + EC = incrementFS(); + return EC; + } + + /// Sets \c CurrentDirIter to an iterator of \c DirPath in the next file + /// system in the list, or leaves it as is (at its end position) if we've + /// already gone through them all. + std::error_code incrementFS() { + while (!FSList.empty()) { + std::error_code EC; + CurrentDirIter = FSList.back()->dir_begin(DirPath, EC); + FSList.pop_back(); + if (EC && EC != errc::no_such_file_or_directory) + return EC; + if (CurrentDirIter != directory_iterator()) + break; // found; + } + return {}; + } + +public: + CombiningDirIterImpl(ArrayRef FileSystems, std::string Dir, + std::error_code &EC) + : FSList(FileSystems.begin(), FileSystems.end()), + DirPath(std::move(Dir)) { + if (!FSList.empty()) { + CurrentDirIter = FSList.back()->dir_begin(DirPath, EC); + FSList.pop_back(); + if (!EC || EC == errc::no_such_file_or_directory) + EC = incrementImpl(true); + } + } + + CombiningDirIterImpl(directory_iterator FirstIter, FileSystemPtr Fallback, + std::string FallbackDir, std::error_code &EC) + : FSList({Fallback}), CurrentDirIter(FirstIter), + DirPath(std::move(FallbackDir)) { + if (!EC || EC == errc::no_such_file_or_directory) + EC = incrementImpl(true); + } + + std::error_code increment() override { return incrementImpl(false); } +}; + +} // namespace + //===-----------------------------------------------------------------------===/ // RealFileSystem implementation //===-----------------------------------------------------------------------===/ @@ -450,72 +534,10 @@ llvm::vfs::detail::DirIterImpl::~DirIterImpl() = default; -namespace { - -class OverlayFSDirIterImpl : public llvm::vfs::detail::DirIterImpl { - OverlayFileSystem &Overlays; - std::string Path; - OverlayFileSystem::iterator CurrentFS; - directory_iterator CurrentDirIter; - llvm::StringSet<> SeenNames; - - std::error_code incrementFS() { - assert(CurrentFS != Overlays.overlays_end() && "incrementing past end"); - ++CurrentFS; - for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) { - std::error_code EC; - CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC); - if (EC && EC != errc::no_such_file_or_directory) - return EC; - if (CurrentDirIter != directory_iterator()) - break; // found - } - return {}; - } - - std::error_code incrementDirIter(bool IsFirstTime) { - assert((IsFirstTime || CurrentDirIter != directory_iterator()) && - "incrementing past end"); - std::error_code EC; - if (!IsFirstTime) - CurrentDirIter.increment(EC); - if (!EC && CurrentDirIter == directory_iterator()) - EC = incrementFS(); - return EC; - } - - std::error_code incrementImpl(bool IsFirstTime) { - while (true) { - std::error_code EC = incrementDirIter(IsFirstTime); - if (EC || CurrentDirIter == directory_iterator()) { - CurrentEntry = directory_entry(); - return EC; - } - CurrentEntry = *CurrentDirIter; - StringRef Name = llvm::sys::path::filename(CurrentEntry.path()); - if (SeenNames.insert(Name).second) - return EC; // name not seen before - } - llvm_unreachable("returned above"); - } - -public: - OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS, - std::error_code &EC) - : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) { - CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC); - EC = incrementImpl(true); - } - - std::error_code increment() override { return incrementImpl(false); } -}; - -} // namespace - directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { return directory_iterator( - std::make_shared(Dir, *this, EC)); + std::make_shared(FSList, Dir.str(), EC)); } void ProxyFileSystem::anchor() {} @@ -1020,48 +1042,47 @@ } } -// FIXME: reuse implementation common with OverlayFSDirIterImpl as these -// iterators are conceptually similar. -class llvm::vfs::VFSFromYamlDirIterImpl +class llvm::vfs::RedirectingFSDirIterImpl : public llvm::vfs::detail::DirIterImpl { std::string Dir; - RedirectingFileSystem::RedirectingDirectoryEntry::iterator Current, End; - - // To handle 'fallthrough' mode we need to iterate at first through - // RedirectingDirectoryEntry and then through ExternalFS. These operations are - // done sequentially, we just need to keep a track of what kind of iteration - // we are currently performing. - - /// Flag telling if we should iterate through ExternalFS or stop at the last - /// RedirectingDirectoryEntry::iterator. - bool IterateExternalFS; - /// Flag telling if we have switched to iterating through ExternalFS. - bool IsExternalFSCurrent = false; - FileSystem &ExternalFS; - directory_iterator ExternalDirIter; - llvm::StringSet<> SeenNames; - - /// To combine multiple iterations, different methods are responsible for - /// different iteration steps. - /// @{ + RedirectingFileSystem::DirectoryEntry::iterator Current, End; - /// Responsible for dispatching between RedirectingDirectoryEntry iteration - /// and ExternalFS iteration. - std::error_code incrementImpl(bool IsFirstTime); - /// Responsible for RedirectingDirectoryEntry iteration. - std::error_code incrementContent(bool IsFirstTime); - /// Responsible for ExternalFS iteration. - std::error_code incrementExternal(); - /// @} + std::error_code incrementImpl(bool IsFirstTime) { + assert((IsFirstTime || Current != End) && "cannot iterate past end"); + if (!IsFirstTime) + ++Current; + if (Current != End) { + SmallString<128> PathStr(Dir); + llvm::sys::path::append(PathStr, (*Current)->getName()); + sys::fs::file_type Type = sys::fs::file_type::type_unknown; + switch ((*Current)->getKind()) { + case RedirectingFileSystem::EK_Directory: + LLVM_FALLTHROUGH; + case RedirectingFileSystem::EK_RemapDirectory: + Type = sys::fs::file_type::directory_file; + break; + case RedirectingFileSystem::EK_File: + Type = sys::fs::file_type::regular_file; + break; + } + CurrentEntry = directory_entry(std::string(PathStr.str()), Type); + } else { + CurrentEntry = directory_entry(); + } + return {}; + }; public: - VFSFromYamlDirIterImpl( - const Twine &Path, - RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin, - RedirectingFileSystem::RedirectingDirectoryEntry::iterator End, - bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC); + RedirectingFSDirIterImpl( + const Twine &Path, RedirectingFileSystem::DirectoryEntry::iterator Begin, + RedirectingFileSystem::DirectoryEntry::iterator End, std::error_code &EC) + : Dir(Path.str()), Current(Begin), End(End) { + EC = incrementImpl(/*IsFirstTime=*/true); + }; - std::error_code increment() override; + std::error_code increment() override { + return incrementImpl(/*IsFirstTime=*/false); + }; }; llvm::ErrorOr @@ -1125,15 +1146,21 @@ directory_iterator RedirectingFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { - ErrorOr E = lookupPath(Dir); + SmallString<64> ExternalRedirect; + ErrorOr E = lookupPath(Dir, ExternalRedirect); + if (!E) { - EC = E.getError(); - if (shouldUseExternalFS() && EC == errc::no_such_file_or_directory) + if (shouldFallBackToExternalFS(E.getError())) return ExternalFS->dir_begin(Dir, EC); + EC = E.getError(); return {}; } - ErrorOr S = status(Dir, *E); + + ErrorOr S = status(Dir, *E, ExternalRedirect); if (!S) { + if (shouldFallBackToExternalFS(S.getError(), *E)) { + return ExternalFS->dir_begin(Dir, EC); + } EC = S.getError(); return {}; } @@ -1143,10 +1170,19 @@ return {}; } - auto *D = cast(*E); - return directory_iterator(std::make_shared( - Dir, D->contents_begin(), D->contents_end(), - /*IterateExternalFS=*/shouldUseExternalFS(), *ExternalFS, EC)); + directory_iterator DirIter; + if (auto *D = cast(*E)) { + DirIter = directory_iterator(std::make_shared( + Dir, D->contents_begin(), D->contents_end(), EC)); + } else { + assert((*E)->getKind() == EK_RemapDirectory); + DirIter = ExternalFS->dir_begin(ExternalRedirect, EC); + } + + if (!shouldUseExternalFS()) + return DirIter; + return directory_iterator(std::make_shared( + DirIter, ExternalFS, Dir.str(), EC)); } void RedirectingFileSystem::setExternalContentsPrefixDir(StringRef PrefixDir) { @@ -1183,7 +1219,7 @@ << "\n"; if (E->getKind() == RedirectingFileSystem::EK_Directory) { - auto *DE = dyn_cast(E); + auto *DE = dyn_cast(E); assert(DE && "Should be a directory"); for (std::unique_ptr &SubEntry : @@ -1284,13 +1320,11 @@ } } } else { // Advance to the next component - auto *DE = dyn_cast( - ParentEntry); + auto *DE = dyn_cast(ParentEntry); for (std::unique_ptr &Content : llvm::make_range(DE->contents_begin(), DE->contents_end())) { auto *DirContent = - dyn_cast( - Content.get()); + dyn_cast(Content.get()); if (DirContent && Name.equals(Content->getName())) return DirContent; } @@ -1298,7 +1332,7 @@ // ... or create a new one std::unique_ptr E = - std::make_unique( + std::make_unique( Name, Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(), 0, 0, 0, file_type::directory_file, sys::fs::all_all)); @@ -1309,8 +1343,7 @@ return ParentEntry; } - auto *DE = - cast(ParentEntry); + auto *DE = cast(ParentEntry); DE->addContent(std::move(E)); return DE->getLastContent(); } @@ -1322,7 +1355,7 @@ StringRef Name = SrcE->getName(); switch (SrcE->getKind()) { case RedirectingFileSystem::EK_Directory: { - auto *DE = cast(SrcE); + auto *DE = cast(SrcE); // Empty directories could be present in the YAML as a way to // describe a file for a current directory after some of its subdir // is parsed. This only leads to redundant walks, ignore it. @@ -1333,14 +1366,21 @@ uniqueOverlayTree(FS, SubEntry.get(), NewParentE); break; } - case RedirectingFileSystem::EK_File: { + case RedirectingFileSystem::EK_RemapDirectory: { assert(NewParentE && "Parent entry must exist"); - auto *FE = cast(SrcE); - auto *DE = - cast(NewParentE); + auto *DR = cast(SrcE); + auto *DE = cast(NewParentE); DE->addContent( - std::make_unique( - Name, FE->getExternalContentsPath(), FE->getUseName())); + std::make_unique( + Name, DR->getExternalContentsPath(), DR->getUseName())); + break; + } + case RedirectingFileSystem::EK_File: { + assert(NewParentE && "Parent entry must exist"); + auto *FE = cast(SrcE); + auto *DE = cast(NewParentE); + DE->addContent(std::make_unique( + Name, FE->getExternalContentsPath(), FE->getUseName())); break; } } @@ -1370,8 +1410,7 @@ SmallString<256> ExternalContentsPath; SmallString<256> Name; yaml::Node *NameValueNode = nullptr; - auto UseExternalName = - RedirectingFileSystem::RedirectingFileEntry::NK_NotSet; + auto UseExternalName = RedirectingFileSystem::NK_NotSet; RedirectingFileSystem::EntryKind Kind; for (auto &I : *M) { @@ -1454,9 +1493,8 @@ bool Val; if (!parseScalarBool(I.getValue(), Val)) return nullptr; - UseExternalName = - Val ? RedirectingFileSystem::RedirectingFileEntry::NK_External - : RedirectingFileSystem::RedirectingFileEntry::NK_Virtual; + UseExternalName = Val ? RedirectingFileSystem::NK_External + : RedirectingFileSystem::NK_Virtual; } else { llvm_unreachable("key missing from Keys"); } @@ -1465,6 +1503,11 @@ if (Stream.failed()) return nullptr; + if (Kind == RedirectingFileSystem::EK_Directory && + !ExternalContentsPath.empty()) { + Kind = RedirectingFileSystem::EK_RemapDirectory; + } + // check for missing keys if (!HasContents) { error(N, "missing key 'contents' or 'external-contents'"); @@ -1475,9 +1518,10 @@ // check invalid configuration if (Kind == RedirectingFileSystem::EK_Directory && - UseExternalName != - RedirectingFileSystem::RedirectingFileEntry::NK_NotSet) { - error(N, "'use-external-name' is not supported for directories"); + ExternalContentsPath.empty() && + UseExternalName != RedirectingFileSystem::NK_NotSet) { + error(N, "'use-external-name' is not supported for directories without " + "'external-contents'"); return nullptr; } @@ -1510,16 +1554,18 @@ std::unique_ptr Result; switch (Kind) { case RedirectingFileSystem::EK_File: - Result = std::make_unique( + Result = std::make_unique( + LastComponent, std::move(ExternalContentsPath), UseExternalName); + break; + case RedirectingFileSystem::EK_RemapDirectory: + Result = std::make_unique( LastComponent, std::move(ExternalContentsPath), UseExternalName); break; case RedirectingFileSystem::EK_Directory: - Result = - std::make_unique( - LastComponent, std::move(EntryArrayContents), - Status("", getNextVirtualUniqueID(), - std::chrono::system_clock::now(), 0, 0, 0, - file_type::directory_file, sys::fs::all_all)); + Result = std::make_unique( + LastComponent, std::move(EntryArrayContents), + Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(), + 0, 0, 0, file_type::directory_file, sys::fs::all_all)); break; } @@ -1533,12 +1579,10 @@ I != E; ++I) { std::vector> Entries; Entries.push_back(std::move(Result)); - Result = - std::make_unique( - *I, std::move(Entries), - Status("", getNextVirtualUniqueID(), - std::chrono::system_clock::now(), 0, 0, 0, - file_type::directory_file, sys::fs::all_all)); + Result = std::make_unique( + *I, std::move(Entries), + Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(), + 0, 0, 0, file_type::directory_file, sys::fs::all_all)); } return Result; } @@ -1725,22 +1769,29 @@ } // Add the file. - auto NewFile = - std::make_unique( - llvm::sys::path::filename(From), To, - UseExternalNames - ? RedirectingFileSystem::RedirectingFileEntry::NK_External - : RedirectingFileSystem::RedirectingFileEntry::NK_Virtual); + auto NewFile = std::make_unique( + llvm::sys::path::filename(From), To, + UseExternalNames ? RedirectingFileSystem::NK_External + : RedirectingFileSystem::NK_Virtual); ToEntry = NewFile.get(); - cast(Parent)->addContent( + cast(Parent)->addContent( std::move(NewFile)); } return FS; } +bool RedirectingFileSystem::shouldFallBackToExternalFS(std::error_code Error, + Entry *SrcEntry) const { + if (SrcEntry && SrcEntry->getKind() != EK_RemapDirectory) + return false; + return shouldUseExternalFS() && + Error == llvm::errc::no_such_file_or_directory; +}; + ErrorOr -RedirectingFileSystem::lookupPath(const Twine &Path_) const { +RedirectingFileSystem::lookupPath(const Twine &Path_, + SmallVectorImpl &ExtRedirect) const { SmallString<256> Path; Path_.toVector(Path); @@ -1759,7 +1810,7 @@ sys::path::const_iterator End = sys::path::end(Path); for (const auto &Root : Roots) { ErrorOr Result = - lookupPath(Start, End, Root.get()); + lookupPath(Start, End, Root.get(), ExtRedirect); if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) return Result; } @@ -1769,14 +1820,27 @@ ErrorOr RedirectingFileSystem::lookupPath(sys::path::const_iterator Start, sys::path::const_iterator End, - RedirectingFileSystem::Entry *From) const { + RedirectingFileSystem::Entry *From, + SmallVectorImpl &ExtRedirect) const { assert(!isTraversalComponent(*Start) && !isTraversalComponent(From->getName()) && "Paths should not contain traversal components"); - StringRef FromName = From->getName(); + // If the matched entry is a remapped directory, sets ExtRedirect to the + // directory's external contents path plus any remaining path components. + auto SetExtRedirect = [&](RedirectingFileSystem::Entry *E) { + auto *DRE = dyn_cast(From); + if (!DRE) + return; + StringRef ExternalPath = DRE->getExternalContentsPath(); + if (!ExternalPath.empty()) { + ExtRedirect.assign(ExternalPath.begin(), ExternalPath.end()); + sys::path::append(ExtRedirect, Start, End); + } + }; // Forward the search to the next component in case this is an empty one. + StringRef FromName = From->getName(); if (!FromName.empty()) { if (!pathComponentMatches(*Start, FromName)) return make_error_code(llvm::errc::no_such_file_or_directory); @@ -1785,22 +1849,27 @@ if (Start == End) { // Match! + SetExtRedirect(From); return From; } } - auto *DE = dyn_cast(From); - if (!DE) + if (isa(From)) return make_error_code(llvm::errc::not_a_directory); + if (isa(From)) { + SetExtRedirect(From); + return From; + } + + auto *DE = cast(From); for (const std::unique_ptr &DirEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) { ErrorOr Result = - lookupPath(Start, End, DirEntry.get()); + lookupPath(Start, End, DirEntry.get(), ExtRedirect); if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) return Result; } - return make_error_code(llvm::errc::no_such_file_or_directory); } @@ -1814,31 +1883,42 @@ } ErrorOr RedirectingFileSystem::status(const Twine &Path, - RedirectingFileSystem::Entry *E) { + RedirectingFileSystem::Entry *E, + StringRef ExternalRedirect) { assert(E != nullptr); - if (auto *F = dyn_cast(E)) { - ErrorOr S = ExternalFS->status(F->getExternalContentsPath()); - assert(!S || S->getName() == F->getExternalContentsPath()); - if (S) - return getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames), - *S); - return S; - } else { // directory - auto *DE = cast(E); - return Status::copyWithNewName(DE->getStatus(), Path); + if (auto *RE = dyn_cast(E)) { + if (ExternalRedirect.empty()) { + assert(RE->getKind() == RedirectingFileSystem::EK_File); + ExternalRedirect = RE->getExternalContentsPath(); + } + + auto S = ExternalFS->status(ExternalRedirect); + if (!S) + return S; + return getRedirectedFileStatus(Path, RE->useExternalName(UseExternalNames), + *S); } + + assert(ExternalRedirect.empty()); + auto *DE = cast(E); + return Status::copyWithNewName(DE->getStatus(), Path); } ErrorOr RedirectingFileSystem::status(const Twine &Path) { - ErrorOr Result = lookupPath(Path); + SmallString<64> ExternalRedirect; + ErrorOr Result = + lookupPath(Path, ExternalRedirect); if (!Result) { - if (shouldUseExternalFS() && - Result.getError() == llvm::errc::no_such_file_or_directory) { + if (shouldFallBackToExternalFS(Result.getError())) { return ExternalFS->status(Path); } return Result.getError(); } - return status(Path, *Result); + + auto S = status(Path, *Result, ExternalRedirect); + if (!S && shouldFallBackToExternalFS(S.getError(), *Result)) + S = ExternalFS->status(Path); + return S; } namespace { @@ -1868,30 +1948,38 @@ ErrorOr> RedirectingFileSystem::openFileForRead(const Twine &Path) { - ErrorOr E = lookupPath(Path); + SmallString<64> ExternalRedirect; + ErrorOr E = + lookupPath(Path, ExternalRedirect); if (!E) { - if (shouldUseExternalFS() && - E.getError() == llvm::errc::no_such_file_or_directory) { + if (shouldFallBackToExternalFS(E.getError())) return ExternalFS->openFileForRead(Path); - } return E.getError(); } - auto *F = dyn_cast(*E); - if (!F) // FIXME: errc::not_a_file? + auto *RE = dyn_cast(*E); + if (!RE) // FIXME: errc::not_a_file? return make_error_code(llvm::errc::invalid_argument); - auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath()); - if (!Result) + if (ExternalRedirect.empty()) { + assert(RE->getKind() == EK_File); + ExternalRedirect = RE->getExternalContentsPath(); + } + + auto Result = ExternalFS->openFileForRead(ExternalRedirect); + if (!Result) { + if (shouldFallBackToExternalFS(Result.getError(), RE)) + Result = ExternalFS->openFileForRead(Path); return Result; + } auto ExternalStatus = (*Result)->status(); if (!ExternalStatus) return ExternalStatus.getError(); // FIXME: Update the status with the name and VFSMapped. - Status S = getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames), - *ExternalStatus); + Status S = getRedirectedFileStatus( + Path, RE->useExternalName(UseExternalNames), *ExternalStatus); return std::unique_ptr( std::make_unique(std::move(*Result), S)); } @@ -1899,19 +1987,28 @@ std::error_code RedirectingFileSystem::getRealPath(const Twine &Path, SmallVectorImpl &Output) const { - ErrorOr Result = lookupPath(Path); + SmallString<64> ExternalRedirect; + ErrorOr Result = + lookupPath(Path, ExternalRedirect); if (!Result) { - if (shouldUseExternalFS() && - Result.getError() == llvm::errc::no_such_file_or_directory) { + if (shouldFallBackToExternalFS(Result.getError())) { return ExternalFS->getRealPath(Path, Output); } return Result.getError(); } - if (auto *F = - dyn_cast(*Result)) { - return ExternalFS->getRealPath(F->getExternalContentsPath(), Output); + if (auto *RE = dyn_cast(*Result)) { + if (ExternalRedirect.empty()) { + assert(RE->getKind() == EK_File); + ExternalRedirect = RE->getExternalContentsPath(); + } + auto P = ExternalFS->getRealPath(ExternalRedirect, Output); + if (!P && shouldFallBackToExternalFS(P, RE)) { + return ExternalFS->getRealPath(Path, Output); + } + return P; } + // Even if there is a directory entry, fall back to ExternalFS if allowed, // because directories don't have a single external contents path. return shouldUseExternalFS() ? ExternalFS->getRealPath(Path, Output) @@ -1933,7 +2030,7 @@ SmallVectorImpl &Entries) { auto Kind = SrcE->getKind(); if (Kind == RedirectingFileSystem::EK_Directory) { - auto *DE = dyn_cast(SrcE); + auto *DE = dyn_cast(SrcE); assert(DE && "Must be a directory"); for (std::unique_ptr &SubEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) { @@ -1941,16 +2038,24 @@ getVFSEntries(SubEntry.get(), Path, Entries); Path.pop_back(); } - return; + } else if (Kind == RedirectingFileSystem::EK_RemapDirectory) { + auto *DR = dyn_cast(SrcE); + assert(DR && "Must be a remap directory"); + SmallString<128> VPath; + for (auto &Comp : Path) + llvm::sys::path::append(VPath, Comp); + Entries.push_back( + YAMLVFSEntry(VPath.c_str(), DR->getExternalContentsPath())); + } else { + assert(Kind == RedirectingFileSystem::EK_File && "Must be a EK_File"); + auto *FE = dyn_cast(SrcE); + assert(FE && "Must be a file"); + SmallString<128> VPath; + for (auto &Comp : Path) + llvm::sys::path::append(VPath, Comp); + Entries.push_back( + YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath())); } - - assert(Kind == RedirectingFileSystem::EK_File && "Must be a EK_File"); - auto *FE = dyn_cast(SrcE); - assert(FE && "Must be a file"); - SmallString<128> VPath; - for (auto &Comp : Path) - llvm::sys::path::append(VPath, Comp); - Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath())); } void vfs::collectVFSFromYAML(std::unique_ptr Buffer, @@ -1962,7 +2067,9 @@ std::unique_ptr VFS = RedirectingFileSystem::create( std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext, std::move(ExternalFS)); - ErrorOr RootE = VFS->lookupPath("/"); + SmallVector ExternalRedirect; + ErrorOr RootE = + VFS->lookupPath("/", ExternalRedirect); if (!RootE) return; SmallVector Components; @@ -2165,76 +2272,6 @@ IsOverlayRelative, OverlayDir); } -VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl( - const Twine &_Path, - RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin, - RedirectingFileSystem::RedirectingDirectoryEntry::iterator End, - bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC) - : Dir(_Path.str()), Current(Begin), End(End), - IterateExternalFS(IterateExternalFS), ExternalFS(ExternalFS) { - EC = incrementImpl(/*IsFirstTime=*/true); -} - -std::error_code VFSFromYamlDirIterImpl::increment() { - return incrementImpl(/*IsFirstTime=*/false); -} - -std::error_code VFSFromYamlDirIterImpl::incrementExternal() { - assert(!(IsExternalFSCurrent && ExternalDirIter == directory_iterator()) && - "incrementing past end"); - std::error_code EC; - if (IsExternalFSCurrent) { - ExternalDirIter.increment(EC); - } else if (IterateExternalFS) { - ExternalDirIter = ExternalFS.dir_begin(Dir, EC); - IsExternalFSCurrent = true; - if (EC && EC != errc::no_such_file_or_directory) - return EC; - EC = {}; - } - if (EC || ExternalDirIter == directory_iterator()) { - CurrentEntry = directory_entry(); - } else { - CurrentEntry = *ExternalDirIter; - } - return EC; -} - -std::error_code VFSFromYamlDirIterImpl::incrementContent(bool IsFirstTime) { - assert((IsFirstTime || Current != End) && "cannot iterate past end"); - if (!IsFirstTime) - ++Current; - while (Current != End) { - SmallString<128> PathStr(Dir); - llvm::sys::path::append(PathStr, (*Current)->getName()); - sys::fs::file_type Type = sys::fs::file_type::type_unknown; - switch ((*Current)->getKind()) { - case RedirectingFileSystem::EK_Directory: - Type = sys::fs::file_type::directory_file; - break; - case RedirectingFileSystem::EK_File: - Type = sys::fs::file_type::regular_file; - break; - } - CurrentEntry = directory_entry(std::string(PathStr.str()), Type); - return {}; - } - return incrementExternal(); -} - -std::error_code VFSFromYamlDirIterImpl::incrementImpl(bool IsFirstTime) { - while (true) { - std::error_code EC = IsExternalFSCurrent ? incrementExternal() - : incrementContent(IsFirstTime); - if (EC || CurrentEntry.path().empty()) - return EC; - StringRef Name = llvm::sys::path::filename(CurrentEntry.path()); - if (SeenNames.insert(Name).second) - return EC; // name not seen before - } - llvm_unreachable("returned above"); -} - vfs::recursive_directory_iterator::recursive_directory_iterator( FileSystem &FS_, const Twine &Path, std::error_code &EC) : FS(&FS_) { Index: llvm/unittests/Support/VirtualFileSystemTest.cpp =================================================================== --- llvm/unittests/Support/VirtualFileSystemTest.cpp +++ llvm/unittests/Support/VirtualFileSystemTest.cpp @@ -1328,6 +1328,7 @@ TEST_F(VFSFromYAMLTest, MappedFiles) { IntrusiveRefCntPtr Lower(new DummyFileSystem()); + Lower->addDirectory("//root/foo/bar"); Lower->addRegularFile("//root/foo/bar/a"); IntrusiveRefCntPtr FS = getFromYAMLString( "{ 'roots': [\n" @@ -1343,6 +1344,17 @@ " 'type': 'file',\n" " 'name': 'file2',\n" " 'external-contents': '//root/foo/b'\n" + " },\n" + " {\n" + " 'type': 'directory',\n" + " 'name': 'mappeddir',\n" + " 'external-contents': '//root/foo/bar'\n" + " },\n" + " {\n" + " 'type': 'directory',\n" + " 'name': 'mappeddir2',\n" + " 'use-external-name': false,\n" + " 'external-contents': '//root/foo/bar'\n" " }\n" " ]\n" "}\n" @@ -1374,18 +1386,102 @@ EXPECT_EQ("//root/foo/bar/a", OpenedS->getName()); EXPECT_TRUE(OpenedS->IsVFSMapped); - // directory + // virtual directory S = O->status("//root/"); ASSERT_FALSE(S.getError()); EXPECT_TRUE(S->isDirectory()); EXPECT_TRUE(S->equivalent(*O->status("//root/"))); // non-volatile UniqueID + // mapped directory + S = O->status("//root/mappeddir"); + ASSERT_FALSE(S.getError()); + EXPECT_TRUE(S->isDirectory()); + EXPECT_TRUE(S->IsVFSMapped); + EXPECT_TRUE(S->equivalent(*O->status("//root/foo/bar"))); + + SLower = O->status("//root/foo/bar"); + EXPECT_EQ("//root/foo/bar", SLower->getName()); + EXPECT_TRUE(S->equivalent(*SLower)); + EXPECT_FALSE(SLower->IsVFSMapped); + + // file in mapped directory + S = O->status("//root/mappeddir/a"); + ASSERT_FALSE(S.getError()); + ASSERT_FALSE(S->isDirectory()); + ASSERT_TRUE(S->IsVFSMapped); + ASSERT_EQ("//root/foo/bar/a", S->getName()); + + // file in mapped directory, with use-external-name=false + S = O->status("//root/mappeddir2/a"); + ASSERT_FALSE(S.getError()); + ASSERT_FALSE(S->isDirectory()); + ASSERT_TRUE(S->IsVFSMapped); + ASSERT_EQ("//root/mappeddir2/a", S->getName()); + + // file contents in mapped directory + OpenedF = O->openFileForRead("//root/mappeddir/a"); + ASSERT_FALSE(OpenedF.getError()); + OpenedS = (*OpenedF)->status(); + ASSERT_FALSE(OpenedS.getError()); + EXPECT_EQ("//root/foo/bar/a", OpenedS->getName()); + EXPECT_TRUE(OpenedS->IsVFSMapped); + + // file contents in mapped directory, with use-external-name=false + OpenedF = O->openFileForRead("//root/mappeddir2/a"); + ASSERT_FALSE(OpenedF.getError()); + OpenedS = (*OpenedF)->status(); + ASSERT_FALSE(OpenedS.getError()); + EXPECT_EQ("//root/mappeddir2/a", OpenedS->getName()); + EXPECT_TRUE(OpenedS->IsVFSMapped); + // broken mapping EXPECT_EQ(O->status("//root/file2").getError(), llvm::errc::no_such_file_or_directory); EXPECT_EQ(0, NumDiagnostics); } +TEST_F(VFSFromYAMLTest, MappedRoot) { + IntrusiveRefCntPtr Lower(new DummyFileSystem()); + Lower->addDirectory("//root/foo/bar"); + Lower->addRegularFile("//root/foo/bar/a"); + IntrusiveRefCntPtr FS = + getFromYAMLString("{ 'roots': [\n" + "{\n" + " 'type': 'directory',\n" + " 'name': '//mappedroot/',\n" + " 'external-contents': '//root/foo/bar'\n" + "}\n" + "]\n" + "}", + Lower); + ASSERT_TRUE(FS.get() != nullptr); + + IntrusiveRefCntPtr O( + new vfs::OverlayFileSystem(Lower)); + O->pushOverlay(FS); + + // file + ErrorOr S = O->status("//mappedroot/a"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("//root/foo/bar/a", S->getName()); + EXPECT_TRUE(S->IsVFSMapped); + + ErrorOr SLower = O->status("//root/foo/bar/a"); + EXPECT_EQ("//root/foo/bar/a", SLower->getName()); + EXPECT_TRUE(S->equivalent(*SLower)); + EXPECT_FALSE(SLower->IsVFSMapped); + + // file after opening + auto OpenedF = O->openFileForRead("//mappedroot/a"); + ASSERT_FALSE(OpenedF.getError()); + auto OpenedS = (*OpenedF)->status(); + ASSERT_FALSE(OpenedS.getError()); + EXPECT_EQ("//root/foo/bar/a", OpenedS->getName()); + EXPECT_TRUE(OpenedS->IsVFSMapped); + + EXPECT_EQ(0, NumDiagnostics); +} + TEST_F(VFSFromYAMLTest, CaseInsensitive) { IntrusiveRefCntPtr Lower(new DummyFileSystem()); Lower->addRegularFile("//root/foo/bar/a"); @@ -1542,7 +1638,17 @@ EXPECT_EQ(nullptr, FS.get()); FS = getFromYAMLRawString("{ 'version':100000, 'roots':[] }", Lower); EXPECT_EQ(nullptr, FS.get()); - EXPECT_EQ(24, NumDiagnostics); + + // both 'external-contents' and 'contents' specified + Lower->addDirectory("//root/external/dir"); + FS = getFromYAMLString( + "{ 'roots':[ \n" + "{ 'type': 'directory', 'name': '//root/A', 'contents': [],\n" + " 'external-contents': '//root/external/dir'}]}", + Lower); + EXPECT_EQ(nullptr, FS.get()); + + EXPECT_EQ(25, NumDiagnostics); } TEST_F(VFSFromYAMLTest, UseExternalName) {