diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -551,14 +551,62 @@ /// Get a globally unique ID for a virtual file or directory. llvm::sys::fs::UniqueID getNextVirtualUniqueID(); -/// Gets a \p FileSystem for a virtual file system described in YAML -/// format. +/// \see getVFSFromYAMLs +/// +/// Returns nullptr if \p Buffer could not be parsed as a YAML containing +/// overlay mappings. std::unique_ptr getVFSFromYAML(std::unique_ptr Buffer, llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext = nullptr, IntrusiveRefCntPtr ExternalFS = getRealFileSystem()); +/// Convenience function to read each file from \p ExternalFS and pass their +/// buffers along to the next overload. Returns a \c FileError if any file +/// could not be read. +Expected> +getVFSFromYAMLs(ArrayRef YAMLOverlayPaths, + IntrusiveRefCntPtr ExternalFS = getRealFileSystem(), + SourceMgr::DiagHandlerTy DiagHandler = nullptr, + void *DiagContext = nullptr); + +/// Gets a \c FileSystem that runs operations on the virtual filesystems +/// described by each buffer in \p Buffers and then the \p ExternalFS if those +/// all fail (dependent on the options below). The order that each is run is +/// the reverse of the list, ie. the last buffer creates the first used +/// \c FileSystem. +/// +/// Returns a \c FileError if any of the buffers are invalid and an +/// \c OverlayFileSystem with just the \p ExternalFS if no buffers were +/// provided. +/// +/// Two extra root configuration options are handled by this method when +/// parsing the YAML of each filesystem: +/// 'fallthrough': +/// 'redirecting-with': +/// +/// These specify the order each filesystem is added to an \c OverlayFileSystem +/// and whether they're added at all - +/// - 'fallthrough': allow "falling through" to the next filesystem +/// - 'fallback': run opertions on the next filesystem before this one. +/// Most useful when specified on the first overlay, which +/// causes the \p ExternalFS to run before it (ie. "fallback" +/// to using the mapped paths only when original fails) +/// - 'redirect-only': skip running operations on any further filesystems, +/// ie. this is the last used filesystem. As with +/// 'fallback', this is more useful when specified on the +/// first overlay to avoid also checking the \p ExternalFS. +/// +/// \see OverlayFileSystem +/// \see RedirectingFileSystem +Expected> +getVFSFromYAMLs(ArrayRef YAMLOverlays, + IntrusiveRefCntPtr ExternalFS = getRealFileSystem(), + SourceMgr::DiagHandlerTy DiagHandler = nullptr, + void *DiagContext = nullptr); + struct YAMLVFSEntry { template YAMLVFSEntry(T1 &&VPath, T2 &&RPath, bool IsDirectory = false) @@ -597,10 +645,6 @@ /// 'case-sensitive': /// 'use-external-names': /// 'overlay-relative': -/// 'fallthrough': -/// 'redirecting-with': /// /// Virtual directories that list their contents are represented as /// \verbatim @@ -860,8 +904,11 @@ /// the path it redirects to in the external file system. ErrorOr lookupPath(StringRef CanonicalPath) const; - /// Parses \p Buffer, which is expected to be in YAML format and - /// returns a virtual file system representing its contents. + /// Parses \p Buffer, which is expected to be in the YAML format described + /// in \c RedirectingFileSystem and returns a virtual file system + /// representing its contents. + /// + /// \see getVFSFromYAMLs static std::unique_ptr create(std::unique_ptr Buffer, SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -2236,9 +2236,132 @@ SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext, IntrusiveRefCntPtr ExternalFS) { - return RedirectingFileSystem::create(std::move(Buffer), DiagHandler, - YAMLFilePath, DiagContext, - std::move(ExternalFS)); + MemoryBufferRef BufferRef {Buffer->getBuffer(), YAMLFilePath}; + auto FS = getVFSFromYAMLs(BufferRef, ExternalFS, DiagHandler, DiagContext); + if (auto Err = FS.takeError()) { + consumeError(std::move(Err)); + return nullptr; + } + return std::move(*FS); +} + +namespace { + +struct VFSResult { + IntrusiveRefCntPtr FS; + YAMLParseResult::RedirectKind Redirection; + + VFSResult(IntrusiveRefCntPtr FS, + YAMLParseResult::RedirectKind Redirection) + : FS(std::move(FS)), Redirection(Redirection) {} + + VFSResult(YAMLParseResult &Result) + : FS(std::move(Result.FS)), Redirection(Result.Redirection) {} +}; + +} // namespace + +static std::unique_ptr +createOverlay(ArrayRef VFSResults) { + if (VFSResults.empty()) + return nullptr; + + auto OverlayFS = std::make_unique(VFSResults.front().FS); + for (const auto &Result : VFSResults.drop_front()) { + OverlayFS->pushOverlay(Result.FS); + } + return OverlayFS; +} + +Expected> +vfs::getVFSFromYAMLs(ArrayRef YAMLOverlayPaths, + IntrusiveRefCntPtr ExternalFS, + SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext) { + SmallVector BufferRefs; + for (StringRef Path : YAMLOverlayPaths) { + BufferRefs.emplace_back(StringRef(), Path); + } + return getVFSFromYAMLs(BufferRefs, std::move(ExternalFS), DiagHandler, + DiagContext); +} + +Expected> +vfs::getVFSFromYAMLs(ArrayRef YAMLOverlays, + IntrusiveRefCntPtr ExternalFS, + SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext) { + if (YAMLOverlays.empty()) + return std::make_unique(std::move(ExternalFS)); + + SourceMgr SM; + SM.setDiagHandler(DiagHandler, DiagContext); + + SmallVector Overlays; + Overlays.emplace_back(ExternalFS, YAMLParseResult::RedirectKind::Fallthrough); + std::unique_ptr OverlayFS = createOverlay(Overlays); + + for (MemoryBufferRef Buffer : YAMLOverlays) { + // Bit of a hack. If the buffer's data is a nullptr (not just empty), + // attempt to read the the file from the current FS. This is to + // differentiate between an actual buffer and one that was made from just + // a path. The paths cannot be read all up front in order to support + // overlay files being defined in a VFS. + std::unique_ptr TempBuffer; + if (Buffer.getBuffer().data() == nullptr) { + ErrorOr> NewBufferOrError = + OverlayFS->getBufferForFile(Buffer.getBufferIdentifier()); + if (!NewBufferOrError) + return createFileError(Buffer.getBufferIdentifier(), + NewBufferOrError.getError()); + TempBuffer = std::move(*NewBufferOrError); + Buffer = MemoryBufferRef(TempBuffer->getBuffer(), + Buffer.getBufferIdentifier()); + } + + Optional Result = + RedirectingFileSystemParser::parse(Buffer, SM, ExternalFS); + if (!Result) + return createFileError(Buffer.getBufferIdentifier(), + llvm::errc::invalid_argument); + + switch (Result->Redirection) { + case YAMLParseResult::RedirectKind::Fallthrough: + // Simple case - add on the FS + Overlays.emplace_back(*Result); + OverlayFS->pushOverlay(Overlays.back().FS); + break; + case YAMLParseResult::RedirectKind::Fallback: + // Fallback implies that the previously added FS should run operations + // before hitting this FS. This doesn't make a whole lot of sense for any + // position other than the beginning where we want the external FS to run + // first. For other positions the overlays could just be specified in the + // opposite order. + // + // This is fallout from the history of \c RedirectingFileSystem. Ideally + // we would just depend on order and have some way to specify the real + // filesystem. + + if (Overlays.back().Redirection != + YAMLParseResult::RedirectKind::RedirectOnly) { + // If the previously inserted overlay is redirect only, don't bother + // adding this one - it would never run anyway (since redirect only + // is necessarily the last FS). + + // Note: Overlays is never empty, it always has at least the + // \c ExternalFS + Overlays.insert(Overlays.end() - 1, VFSResult(*Result)); + OverlayFS = createOverlay(Overlays); + } + break; + case YAMLParseResult::RedirectKind::RedirectOnly: + // This is now the last FS, clear the rest if there were any + Overlays.clear(); + Overlays.emplace_back(*Result); + OverlayFS = createOverlay(Overlays); + break; + } + } + + return OverlayFS; } static void getVFSEntries(RedirectingFileSystem::Entry *SrcE, diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp --- a/llvm/unittests/Support/VirtualFileSystemTest.cpp +++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp @@ -14,6 +14,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Testing/Support/Error.h" #include "llvm/Testing/Support/SupportHelpers.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -40,6 +41,7 @@ llvm_unreachable("unimplemented"); } std::error_code close() override { return std::error_code(); } + void setPath(const Twine &Path) override { S = S.copyWithNewName(S, Path); } }; class DummyFileSystem : public vfs::FileSystem { @@ -1701,30 +1703,21 @@ EXPECT_EQ(0, NumDiagnostics); } -TEST_F(VFSFromYAMLTest, ReturnsRequestedPathVFSMiss) { +TEST_F(VFSFromYAMLTest, RelativeVFSPathMiss) { IntrusiveRefCntPtr BaseFS( new vfs::InMemoryFileSystem); BaseFS->addFile("//root/foo/a", 0, MemoryBuffer::getMemBuffer("contents of a")); ASSERT_FALSE(BaseFS->setCurrentWorkingDirectory("//root/foo")); + auto RemappedFS = vfs::RedirectingFileSystem::create( {}, /*UseExternalNames=*/false, *BaseFS); - auto OpenedF = RemappedFS->openFileForRead("a"); + auto OpenedF = BaseFS->openFileForRead("a"); ASSERT_FALSE(OpenedF.getError()); - llvm::ErrorOr Name = (*OpenedF)->getName(); - ASSERT_FALSE(Name.getError()); - EXPECT_EQ("a", Name.get()); - auto OpenedS = (*OpenedF)->status(); - ASSERT_FALSE(OpenedS.getError()); - EXPECT_EQ("a", OpenedS->getName()); - EXPECT_FALSE(OpenedS->IsVFSMapped); - - auto DirectS = RemappedFS->status("a"); - ASSERT_FALSE(DirectS.getError()); - EXPECT_EQ("a", DirectS->getName()); - EXPECT_FALSE(DirectS->IsVFSMapped); + OpenedF = RemappedFS->openFileForRead("a"); + ASSERT_TRUE(OpenedF.getError()); EXPECT_EQ(0, NumDiagnostics); } @@ -2441,7 +2434,7 @@ } TEST_F(VFSFromYAMLTest, WorkingDirectory) { - IntrusiveRefCntPtr Lower(new DummyFileSystem()); + IntrusiveRefCntPtr Lower(new ErrorDummyFileSystem()); Lower->addDirectory("//root/"); Lower->addDirectory("//root/foo"); Lower->addRegularFile("//root/foo/a"); @@ -2463,6 +2456,7 @@ "}", Lower); ASSERT_NE(FS.get(), nullptr); + std::error_code EC = FS->setCurrentWorkingDirectory("//root/bar"); ASSERT_FALSE(EC); @@ -2491,6 +2485,14 @@ ASSERT_TRUE(WorkingDir); EXPECT_EQ(*WorkingDir, "//root/"); + Status = FS->status("bar/a"); + ASSERT_FALSE(Status.getError()); + EXPECT_TRUE(Status->exists()); + + // CWD wasn't set on Lower (it just errors instead), so this should fail + Status = FS->status("foo/a"); + ASSERT_TRUE(Status.getError()); + EC = FS->setCurrentWorkingDirectory("bar"); ASSERT_FALSE(EC); WorkingDir = FS->getCurrentWorkingDirectory(); @@ -2580,43 +2582,6 @@ EXPECT_TRUE(Status->exists()); } -TEST_F(VFSFromYAMLTest, WorkingDirectoryFallthroughInvalid) { - IntrusiveRefCntPtr Lower(new ErrorDummyFileSystem()); - Lower->addDirectory("//root/"); - Lower->addDirectory("//root/foo"); - Lower->addRegularFile("//root/foo/a"); - Lower->addRegularFile("//root/foo/b"); - Lower->addRegularFile("//root/c"); - IntrusiveRefCntPtr FS = getFromYAMLString( - "{ 'use-external-names': false,\n" - " 'roots': [\n" - "{\n" - " 'type': 'directory',\n" - " 'name': '//root/bar',\n" - " 'contents': [ {\n" - " 'type': 'file',\n" - " 'name': 'a',\n" - " 'external-contents': '//root/foo/a'\n" - " }\n" - " ]\n" - "}\n" - "]\n" - "}", - Lower); - ASSERT_NE(FS.get(), nullptr); - std::error_code EC = FS->setCurrentWorkingDirectory("//root/"); - ASSERT_FALSE(EC); - ASSERT_NE(FS.get(), nullptr); - - llvm::ErrorOr Status = FS->status("bar/a"); - ASSERT_FALSE(Status.getError()); - EXPECT_TRUE(Status->exists()); - - Status = FS->status("foo/a"); - ASSERT_FALSE(Status.getError()); - EXPECT_TRUE(Status->exists()); -} - TEST_F(VFSFromYAMLTest, VirtualWorkingDirectory) { IntrusiveRefCntPtr Lower(new ErrorDummyFileSystem()); Lower->addDirectory("//root/"); @@ -3048,3 +3013,190 @@ ASSERT_FALSE(S.getError()); EXPECT_EQ("b/c", S->getName()); } + +static std::unique_ptr +getVFSOrNull(ArrayRef YAMLOverlays, + IntrusiveRefCntPtr ExternalFS) { + SmallVector OverlayRefs; + for (const auto &Overlay : YAMLOverlays) { + OverlayRefs.emplace_back(Overlay, ""); + } + + auto ExpectedFS = vfs::getVFSFromYAMLs(OverlayRefs, ExternalFS); + if (auto Err = ExpectedFS.takeError()) { + consumeError(std::move(Err)); + return nullptr; + } + return std::move(*ExpectedFS); +} + +static std::string createSimpleOverlay(StringRef RedirectKind, StringRef From, + StringRef To) { + return ("{\n" + " 'version': 0,\n" + " 'redirecting-with': '" + + RedirectKind + + "'\n" + " 'roots': [\n" + " {\n" + " 'type': 'directory-remap',\n" + " 'name': '" + + From + + "',\n" + " 'external-contents': '" + + To + + "',\n" + " }]\n" + " }" + " ]") + .str(); +} + +// Make sure that overlays are not transitive. Given A -> B and B -> C, if a +// file in A is requested, it should not end up mapping to C. +TEST(VFSFromYAMLsTest, NotTransitive) { + auto C = makeIntrusiveRefCnt(); + C->addDirectory("/real/c"); + C->addRegularFile("/real/c/f"); + + SmallVector Overlays; + Overlays.push_back(createSimpleOverlay("fallthrough", "/b", "/real/c")); + Overlays.push_back(createSimpleOverlay("fallthrough", "/a", "/b")); + auto FS = getVFSOrNull(Overlays, C); + ASSERT_TRUE(FS); + + auto S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real/c/f", S->getName()); + + S = FS->status("/a/f"); + EXPECT_TRUE(S.getError()); +} + +// When fallback is the first overlay, the external FS should be checked before +// it. Given B -> A, A should only be used if the file does not exist in B. +TEST(VFSFromYAMLsTest, FallbackChecksExternalFirst) { + auto AOnly = makeIntrusiveRefCnt(); + AOnly->addDirectory("/a"); + AOnly->addRegularFile("/a/f"); + + auto BOnly = makeIntrusiveRefCnt(); + BOnly->addDirectory("/b"); + BOnly->addRegularFile("/b/f"); + + auto Both = makeIntrusiveRefCnt(); + Both->addDirectory("/a"); + Both->addRegularFile("/a/f"); + Both->addDirectory("/b"); + Both->addRegularFile("/b/f"); + + auto FallbackOverlay = createSimpleOverlay("fallback", "/b", "/a"); + + auto FS = getVFSOrNull(FallbackOverlay, AOnly); + ASSERT_TRUE(FS); + auto S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/f", S->getName()); + + FS = getVFSOrNull(FallbackOverlay, BOnly); + ASSERT_TRUE(FS); + S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/b/f", S->getName()); + + FS = getVFSOrNull(FallbackOverlay, Both); + ASSERT_TRUE(FS); + S = FS->status("/b/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/b/f", S->getName()); +} + +// Ensure the last overlay with redirect-only specified is the final FS +TEST(VFSFromYAMLsTest, RedirectOnlyIsFinalFS) { + auto Real = makeIntrusiveRefCnt(); + Real->addDirectory("/real"); + Real->addRegularFile("/real/f"); + Real->setCurrentWorkingDirectory("/real"); + + SmallVector Overlays; + Overlays.push_back(createSimpleOverlay("redirect-only", "/ro1", "/real")); + Overlays.push_back(createSimpleOverlay("fallthrough", "/ft1", "/real")); + Overlays.push_back(createSimpleOverlay("redirect-only", "/ro2", "/real")); + Overlays.push_back(createSimpleOverlay("fallback", "/fb", "/real")); + Overlays.push_back(createSimpleOverlay("fallthrough", "/ft2", "/real")); + auto FS = getVFSOrNull(Overlays, Real); + ASSERT_TRUE(FS); + + // Should have the same CWD as the external FS we passed down, even if that + // FS isn't actually being used in the overlay any more. + auto CWD = FS->getCurrentWorkingDirectory(); + ASSERT_FALSE(CWD.getError()); + EXPECT_EQ(*CWD, "/real"); + + // Only //ft2/f and //ro2/f should be valid: + // - //ro1 and //ft1 are specified before //ro2 so never run + // - //fb is specified after, but it's set to fallback and hence //ro2 + // should run first (and not continue) + + auto S = FS->status("/ro1/f"); + EXPECT_TRUE(S.getError()); + + S = FS->status("/ft1/f"); + EXPECT_TRUE(S.getError()); + + S = FS->status("/fb/f"); + EXPECT_TRUE(S.getError()); + + S = FS->status("/ro2/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real/f", S->getName()); + + S = FS->status("/ft2/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/real/f", S->getName()); +} + +// Ensure overlays are read from the OverlayFS built so far +TEST(VFSFromYAMLsTest, OverlayFromVFS) { + std::string FT1 = createSimpleOverlay("fallthrough", "/vfs1", "/a"); + std::string FT2 = createSimpleOverlay("fallthrough", "/vfs2", "/b"); + std::string FT3 = createSimpleOverlay("fallthrough", "/vfs3", "/c"); + std::string RO = createSimpleOverlay("redirect-only", "/vfs", "/a"); + + auto Real = makeIntrusiveRefCnt(); + Real->addFile("/ft1.yaml", 0, MemoryBuffer::getMemBuffer(FT1)); + Real->addFile("/ft2.yaml", 0, MemoryBuffer::getMemBuffer(FT2)); + Real->addFile("/a/ft3.yaml", 0, MemoryBuffer::getMemBuffer(FT3)); + Real->addFile("/ro.yaml", 0, MemoryBuffer::getMemBuffer(RO)); + Real->addFile("/a/f", 0, MemoryBuffer::getMemBuffer("a")); + Real->addFile("/b/f", 0, MemoryBuffer::getMemBuffer("b")); + Real->addFile("/c/f", 0, MemoryBuffer::getMemBuffer("c")); + + // 3 to check we're not just using the last overlay + auto ExpectedFS = + vfs::getVFSFromYAMLs({"/ft1.yaml", "/ft2.yaml", "/vfs1/ft3.yaml"}, Real); + ASSERT_THAT_EXPECTED(ExpectedFS, Succeeded()); + auto FS = std::move(*ExpectedFS); + ASSERT_TRUE(FS); + auto S = FS->status("/vfs1/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/f", S->getName()); + S = FS->status("/vfs2/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/b/f", S->getName()); + S = FS->status("/vfs3/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/c/f", S->getName()); + + // If redirect-only is given, any further overlays *must* be specified by it + ExpectedFS = vfs::getVFSFromYAMLs({"/ro.yaml", "/a/ft3.yaml"}, Real); + EXPECT_THAT_EXPECTED(ExpectedFS, Failed()); + + ExpectedFS = vfs::getVFSFromYAMLs({"/ro.yaml", "/vfs/ft3.yaml"}, Real); + ASSERT_THAT_EXPECTED(ExpectedFS, Succeeded()); + FS = std::move(*ExpectedFS); + ASSERT_TRUE(FS); + S = FS->status("/vfs/f"); + ASSERT_FALSE(S.getError()); + EXPECT_EQ("/a/f", S->getName()); +}