diff --git a/llvm/include/llvm/Support/VirtualOutputBackend.h b/llvm/include/llvm/Support/VirtualOutputBackend.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputBackend.h @@ -0,0 +1,62 @@ +//===- VirtualOutputBackend.h - Output virtualization -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H +#define LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/VirtualOutputFile.h" + +namespace llvm::vfs { + +/// Interface for virtualized outputs. +/// +/// If virtual functions are added here, also add them to \a +/// ProxyOutputBackend. +class OutputBackend : public RefCountedBase { + virtual void anchor(); + +public: + /// Get a backend that points to the same destination as this one but that + /// has independent settings. + /// + /// Not thread-safe, but all operations are thread-safe when performed on + /// separate clones of the same backend. + IntrusiveRefCntPtr clone() const { return cloneImpl(); } + + /// Create a file. If \p Config is \c std::nullopt, uses the backend's default + /// OutputConfig (may match \a OutputConfig::OutputConfig(), or may + /// have been customized). + /// + /// Thread-safe. + Expected + createFile(const Twine &Path, + std::optional Config = std::nullopt); + +protected: + /// Must be thread-safe. Virtual function has a different name than \a + /// clone() so that implementations can override the return value. + virtual IntrusiveRefCntPtr cloneImpl() const = 0; + + /// Create a file for \p Path. Must be thread-safe. + /// + /// \pre \p Config is valid or std::nullopt. + virtual Expected> + createFileImpl(StringRef Path, std::optional Config) = 0; + + OutputBackend() = default; + +public: + virtual ~OutputBackend() = default; +}; + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H diff --git a/llvm/include/llvm/Support/VirtualOutputBackends.h b/llvm/include/llvm/Support/VirtualOutputBackends.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputBackends.h @@ -0,0 +1,110 @@ +//===- VirtualOutputBackends.h - Virtual output backends --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H +#define LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" + +namespace llvm::vfs { + +/// Create a backend that ignores all output. +IntrusiveRefCntPtr makeNullOutputBackend(); + +/// Make a backend where \a OutputBackend::createFile() forwards to +/// \p UnderlyingBackend when \p Filter is true, and otherwise returns a +/// \a NullOutput. +IntrusiveRefCntPtr makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + std::function)> Filter); + +/// Create a backend that forwards \a OutputBackend::createFile() to both \p +/// Backend1 and \p Backend2 and sends content to both places. +IntrusiveRefCntPtr +makeMirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2); + +/// A helper class for proxying another backend, with the default +/// implementation to forward to the underlying backend. +class ProxyOutputBackend : public OutputBackend { + void anchor() override; + +protected: + // Require subclass to implement cloneImpl(). + // + // IntrusiveRefCntPtr cloneImpl() const override; + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + OutputFile File; + if (Error E = UnderlyingBackend->createFile(Path, Config).moveInto(File)) + return std::move(E); + return File.takeImpl(); + } + + OutputBackend &getUnderlyingBackend() const { return *UnderlyingBackend; } + +public: + ProxyOutputBackend(IntrusiveRefCntPtr UnderlyingBackend) + : UnderlyingBackend(std::move(UnderlyingBackend)) { + assert(this->UnderlyingBackend && "Expected non-null backend"); + } + +private: + IntrusiveRefCntPtr UnderlyingBackend; +}; + +/// An output backend that creates files on disk, wrapping APIs in sys::fs. +class OnDiskOutputBackend : public OutputBackend { + void anchor() override; + +protected: + IntrusiveRefCntPtr cloneImpl() const override { + return clone(); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override; + +public: + /// Resolve an absolute path. + Error makeAbsolute(SmallVectorImpl &Path) const; + + /// On disk output settings. + struct OutputSettings { + /// Register output files to be deleted if a signal is received. Also + /// disabled for outputs with \a OutputConfig::getNoDiscardOnSignal(). + bool DisableRemoveOnSignal = false; + + /// Disable temporary files. Also disabled for outputs with \a + /// OutputConfig::getNoAtomicWrite(). + bool DisableTemporaries = false; + + // Default configuration for this backend. + OutputConfig DefaultConfig; + }; + + IntrusiveRefCntPtr clone() const { + auto Clone = makeIntrusiveRefCnt(); + Clone->Settings = Settings; + return Clone; + } + + OnDiskOutputBackend() = default; + + /// Settings for this backend. + /// + /// Access is not thread-safe. + OutputSettings Settings; +}; + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.h b/llvm/include/llvm/Support/VirtualOutputConfig.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputConfig.h @@ -0,0 +1,91 @@ +//===- VirtualOutputConfig.h - Virtual output configuration -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H +#define LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H + +#include "llvm/Support/Compiler.h" +#include + +namespace llvm { + +class raw_ostream; + +namespace sys { +namespace fs { +enum OpenFlags : unsigned; +} // end namespace fs +} // end namespace sys + +namespace vfs { + +namespace detail { +/// Unused and empty base class to allow OutputConfig constructor to be +/// constexpr, with commas before every field's initializer. +struct EmptyBaseClass {}; +} // namespace detail + +/// Full configuration for an output for use by the \a OutputBackend. Each +/// configuration flag is either \c true or \c false. +struct OutputConfig : detail::EmptyBaseClass { +public: + void print(raw_ostream &OS) const; + void dump() const; + +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \ + constexpr bool get##NAME() const { return NAME; } \ + constexpr bool getNo##NAME() const { return !NAME; } \ + constexpr OutputConfig &set##NAME(bool Value) { \ + NAME = Value; \ + return *this; \ + } \ + constexpr OutputConfig &set##NAME() { return set##NAME(true); } \ + constexpr OutputConfig &setNo##NAME() { return set##NAME(false); } +#include "llvm/Support/VirtualOutputConfig.def" + + constexpr OutputConfig &setBinary() { return setNoText().setNoCRLF(); } + constexpr OutputConfig &setTextWithCRLF() { return setText().setCRLF(); } + constexpr OutputConfig &setTextWithCRLF(bool Value) { + return Value ? setText().setCRLF() : setBinary(); + } + constexpr bool getTextWithCRLF() { return getText() && getCRLF(); } + constexpr bool getBinary() { return !getText(); } + + /// Updates Text and CRLF flags based on \a sys::fs::OF_Text and \a + /// sys::fs::OF_CRLF in \p Flags. Rejects CRLF without Text (calling + /// \a setBinary()). + OutputConfig &setOpenFlags(const sys::fs::OpenFlags &Flags); + + constexpr OutputConfig() + : EmptyBaseClass() +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) , NAME(DEFAULT) +#include "llvm/Support/VirtualOutputConfig.def" + { + } + + constexpr bool operator==(OutputConfig RHS) const { +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \ + if (NAME != RHS.NAME) \ + return false; +#include "llvm/Support/VirtualOutputConfig.def" + return true; + } + constexpr bool operator!=(OutputConfig RHS) const { return !operator==(RHS); } + +private: +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) bool NAME : 1; +#include "llvm/Support/VirtualOutputConfig.def" +}; + +} // namespace vfs + +raw_ostream &operator<<(raw_ostream &OS, vfs::OutputConfig Config); + +} // namespace llvm + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.def b/llvm/include/llvm/Support/VirtualOutputConfig.def new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputConfig.def @@ -0,0 +1,23 @@ +//===- VirtualOutputConfig.def - Virtual output config defs -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef HANDLE_OUTPUT_CONFIG_FLAG +#error "Missing macro definition of HANDLE_OUTPUT_CONFIG_FLAG" +#endif + +// Define HANDLE_OUTPUT_CONFIG_FLAG before including. +// +// #define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) + +HANDLE_OUTPUT_CONFIG_FLAG(Text, false) // OF_Text. +HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF. +HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal. +HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries. +HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true) + +#undef HANDLE_OUTPUT_CONFIG_FLAG diff --git a/llvm/include/llvm/Support/VirtualOutputError.h b/llvm/include/llvm/Support/VirtualOutputError.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputError.h @@ -0,0 +1,134 @@ +//===- VirtualOutputError.h - Errors for output virtualization --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTERROR_H +#define LLVM_SUPPORT_VIRTUALOUTPUTERROR_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualOutputConfig.h" + +namespace llvm::vfs { + +const std::error_category &output_category(); + +enum class OutputErrorCode { + // Error code 0 is absent. Use std::error_code() instead. + not_closed = 1, + invalid_config, + already_closed, + has_open_proxy, +}; + +inline std::error_code make_error_code(OutputErrorCode EV) { + return std::error_code(static_cast(EV), output_category()); +} + +/// Error related to an \a OutputFile. Derives from \a ECError and adds \a +/// getOutputPath(). +class OutputError : public ErrorInfo { + void anchor() override; + +public: + StringRef getOutputPath() const { return OutputPath; } + void log(raw_ostream &OS) const override { + OS << getOutputPath() << ": "; + ECError::log(OS); + } + + // Used by ErrorInfo::classID. + static char ID; + + OutputError(const Twine &OutputPath, std::error_code EC) + : ErrorInfo(EC), OutputPath(OutputPath.str()) { + assert(EC && "Cannot create OutputError from success EC"); + } + + OutputError(const Twine &OutputPath, OutputErrorCode EV) + : ErrorInfo(make_error_code(EV)), + OutputPath(OutputPath.str()) { + assert(EC && "Cannot create OutputError from success EC"); + } + +private: + std::string OutputPath; +}; + +/// Return \a Error::success() or use \p OutputPath to create an \a +/// OutputError, depending on \p EC. +inline Error convertToOutputError(const Twine &OutputPath, std::error_code EC) { + if (EC) + return make_error(OutputPath, EC); + return Error::success(); +} + +/// Error related to an OutputConfig for an \a OutputFile. Derives from \a +/// OutputError and adds \a getConfig(). +class OutputConfigError : public ErrorInfo { + void anchor() override; + +public: + OutputConfig getConfig() const { return Config; } + void log(raw_ostream &OS) const override { + OutputError::log(OS); + OS << ": " << Config; + } + + // Used by ErrorInfo::classID. + static char ID; + + OutputConfigError(OutputConfig Config, const Twine &OutputPath) + : ErrorInfo( + OutputPath, OutputErrorCode::invalid_config), + Config(Config) {} + +private: + OutputConfig Config; +}; + +/// Error related to a temporary file for an \a OutputFile. Derives from \a +/// OutputError and adds \a getTempPath(). +class TempFileOutputError : public ErrorInfo { + void anchor() override; + +public: + StringRef getTempPath() const { return TempPath; } + void log(raw_ostream &OS) const override { + OS << getTempPath() << " => "; + OutputError::log(OS); + } + + // Used by ErrorInfo::classID. + static char ID; + + TempFileOutputError(const Twine &TempPath, const Twine &OutputPath, + std::error_code EC) + : ErrorInfo(OutputPath, EC), + TempPath(TempPath.str()) {} + + TempFileOutputError(const Twine &TempPath, const Twine &OutputPath, + OutputErrorCode EV) + : ErrorInfo(OutputPath, EV), + TempPath(TempPath.str()) {} + +private: + std::string TempPath; +}; + +/// Return \a Error::success() or use \p TempPath and \p OutputPath to create a +/// \a TempFileOutputError, depending on \p EC. +inline Error convertToTempFileOutputError(const Twine &TempPath, + const Twine &OutputPath, + std::error_code EC) { + if (EC) + return make_error(TempPath, OutputPath, EC); + return Error::success(); +} + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTERROR_H diff --git a/llvm/include/llvm/Support/VirtualOutputFile.h b/llvm/include/llvm/Support/VirtualOutputFile.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputFile.h @@ -0,0 +1,163 @@ +//===- VirtualOutputFile.h - Output file virtualization ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTFILE_H +#define LLVM_SUPPORT_VIRTUALOUTPUTFILE_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ExtensibleRTTI.h" +#include "llvm/Support/VirtualOutputError.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm::vfs { + +class OutputFileImpl : public RTTIExtends { + void anchor() override; + +public: + static char ID; + virtual ~OutputFileImpl() = default; + + virtual Error keep() = 0; + virtual Error discard() = 0; + virtual raw_pwrite_stream &getOS() = 0; +}; + +class NullOutputFileImpl final + : public RTTIExtends { + void anchor() override; + +public: + static char ID; + Error keep() final { return Error::success(); } + Error discard() final { return Error::success(); } + raw_pwrite_stream &getOS() final { return OS; } + +private: + raw_null_ostream OS; +}; + +/// A virtualized output file that writes to a specific backend. +/// +/// One of \a keep(), \a discard(), or \a discardOnDestroy() must be called +/// before destruction. +class OutputFile { +public: + StringRef getPath() const { return Path; } + + /// Check if \a keep() or \a discard() has already been called. + bool isOpen() const { return bool(Impl); } + + explicit operator bool() const { return isOpen(); } + + raw_pwrite_stream &getOS() { + assert(isOpen() && "Expected open output stream"); + return Impl->getOS(); + } + operator raw_pwrite_stream &() { return getOS(); } + template raw_ostream &operator<<(T &&V) { + return getOS() << std::forward(V); + } + + /// Keep an output. Errors if this fails. + /// + /// If it has already been closed, calls \a report_fatal_error(). + /// + /// If there's an open proxy from \a createProxy(), calls \a discard() to + /// clean up temporaries followed by \a report_fatal_error(). + Error keep(); + + /// Discard an output, cleaning up any temporary state. Errors if clean-up + /// fails. + /// + /// If it has already been closed, calls \a report_fatal_error(). + Error discard(); + + /// Discard the output when destroying it if it's still open, sending the + /// result to \a Handler. + void discardOnDestroy(unique_function Handler) { + DiscardOnDestroyHandler = std::move(Handler); + } + + /// Create a proxy stream for clients that need to pass an owned stream to a + /// producer. Errors if there's already a proxy. The proxy must be deleted + /// before calling \a keep(). The proxy will crash if it's written to after + /// calling \a discard(). + Expected> createProxy(); + + bool hasOpenProxy() const { return OpenProxy; } + + /// Take the implementation. + /// + /// \pre \a hasOpenProxy() is false. + /// \pre \a discardOnDestroy() has not been called. + std::unique_ptr takeImpl() { + assert(!hasOpenProxy() && "Unexpected open proxy"); + assert(!DiscardOnDestroyHandler && "Unexpected discard handler"); + return std::move(Impl); + } + + /// Check whether this is a null output file. + bool isNull() const { return Impl && isa(*Impl); } + + OutputFile() = default; + + explicit OutputFile(const Twine &Path, std::unique_ptr Impl) + : Path(Path.str()), Impl(std::move(Impl)) { + assert(this->Impl && "Expected open output file"); + } + + ~OutputFile() { destroy(); } + OutputFile(OutputFile &&O) { moveFrom(O); } + OutputFile &operator=(OutputFile &&O) { + destroy(); + return moveFrom(O); + } + +private: + /// Destroy \a Impl. Reports fatal error if the file is open and there's no + /// handler from \a discardOnDestroy(). + void destroy(); + OutputFile &moveFrom(OutputFile &O) { + Path = std::move(O.Path); + Impl = std::move(O.Impl); + DiscardOnDestroyHandler = std::move(O.DiscardOnDestroyHandler); + OpenProxy = O.OpenProxy; + O.OpenProxy = nullptr; + return *this; + } + + std::string Path; + std::unique_ptr Impl; + unique_function DiscardOnDestroyHandler; + + class TrackedProxy; + TrackedProxy *OpenProxy = nullptr; +}; + +/// Update \p File to silently discard itself if it's still open when it's +/// destroyed. +inline void consumeDiscardOnDestroy(OutputFile &File) { + File.discardOnDestroy(consumeError); +} + +/// Update \p File to silently discard itself if it's still open when it's +/// destroyed. +inline Expected consumeDiscardOnDestroy(Expected File) { + if (File) + consumeDiscardOnDestroy(*File); + return File; +} + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTFILE_H diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -234,6 +234,11 @@ UnicodeNameToCodepointGenerated.cpp VersionTuple.cpp VirtualFileSystem.cpp + VirtualOutputBackend.cpp + VirtualOutputBackends.cpp + VirtualOutputConfig.cpp + VirtualOutputError.cpp + VirtualOutputFile.cpp WithColor.cpp YAMLParser.cpp YAMLTraits.cpp diff --git a/llvm/lib/Support/VirtualOutputBackend.cpp b/llvm/lib/Support/VirtualOutputBackend.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputBackend.cpp @@ -0,0 +1,37 @@ +//===- VirtualOutputBackend.cpp - Virtualize compiler outputs -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements vfs::OutputBackend. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackend.h" + +using namespace llvm; +using namespace llvm::vfs; + +void OutputBackend::anchor() {} + +Expected +OutputBackend::createFile(const Twine &Path_, + std::optional Config) { + SmallString<128> Path; + Path_.toVector(Path); + + if (Config) { + // Check for invalid configs. + if (!Config->getText() && Config->getCRLF()) + return make_error(*Config, Path); + } + + std::unique_ptr Impl; + if (Error E = createFileImpl(Path, Config).moveInto(Impl)) + return std::move(E); + assert(Impl && "Expected valid Impl or Error"); + return OutputFile(Path, std::move(Impl)); +} diff --git a/llvm/lib/Support/VirtualOutputBackends.cpp b/llvm/lib/Support/VirtualOutputBackends.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputBackends.cpp @@ -0,0 +1,409 @@ +//===- VirtualOutputBackends.cpp - Virtual output backends ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements vfs::OutputBackend. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::vfs; + +void ProxyOutputBackend::anchor() {} +void OnDiskOutputBackend::anchor() {} + +IntrusiveRefCntPtr vfs::makeNullOutputBackend() { + struct NullOutputBackend : public OutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + return const_cast(this); + } + Expected> + createFileImpl(StringRef Path, std::optional) override { + return std::make_unique(); + } + }; + + return makeIntrusiveRefCnt(); +} + +IntrusiveRefCntPtr vfs::makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + std::function)> Filter) { + struct FilteringOutputBackend : public ProxyOutputBackend { + Expected> + createFileImpl(StringRef Path, + std::optional Config) override { + if (Filter(Path, Config)) + return ProxyOutputBackend::createFileImpl(Path, Config); + return std::make_unique(); + } + + IntrusiveRefCntPtr cloneImpl() const override { + return makeIntrusiveRefCnt( + getUnderlyingBackend().clone(), Filter); + } + + FilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + std::function)> Filter) + : ProxyOutputBackend(std::move(UnderlyingBackend)), + Filter(std::move(Filter)) { + assert(this->Filter && "Expected a non-null function"); + } + std::function)> Filter; + }; + + return makeIntrusiveRefCnt( + std::move(UnderlyingBackend), std::move(Filter)); +} + +IntrusiveRefCntPtr +vfs::makeMirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) { + struct ProxyOutputBackend1 : public ProxyOutputBackend { + using ProxyOutputBackend::ProxyOutputBackend; + }; + struct ProxyOutputBackend2 : public ProxyOutputBackend { + using ProxyOutputBackend::ProxyOutputBackend; + }; + struct MirroringOutput final : public OutputFileImpl, raw_pwrite_stream { + Error keep() final { + flush(); + return joinErrors(F1->keep(), F2->keep()); + } + Error discard() final { + flush(); + return joinErrors(F1->discard(), F2->discard()); + } + raw_pwrite_stream &getOS() final { return *this; } + + void write_impl(const char *Ptr, size_t Size) override { + F1->getOS().write(Ptr, Size); + F2->getOS().write(Ptr, Size); + } + void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override { + this->flush(); + F1->getOS().pwrite(Ptr, Size, Offset); + F2->getOS().pwrite(Ptr, Size, Offset); + } + uint64_t current_pos() const override { return F1->getOS().tell(); } + size_t preferred_buffer_size() const override { + return PreferredBufferSize; + } + void reserveExtraSpace(uint64_t ExtraSize) override { + F1->getOS().reserveExtraSpace(ExtraSize); + F2->getOS().reserveExtraSpace(ExtraSize); + } + bool is_displayed() const override { + return F1->getOS().is_displayed() && F2->getOS().is_displayed(); + } + bool has_colors() const override { + return F1->getOS().has_colors() && F2->getOS().has_colors(); + } + void enable_colors(bool enable) override { + raw_pwrite_stream::enable_colors(enable); + F1->getOS().enable_colors(enable); + F2->getOS().enable_colors(enable); + } + + MirroringOutput(std::unique_ptr F1, + std::unique_ptr F2) + : PreferredBufferSize(std::max(F1->getOS().GetBufferSize(), + F1->getOS().GetBufferSize())), + F1(std::move(F1)), F2(std::move(F2)) { + // Don't double buffer. + this->F1->getOS().SetUnbuffered(); + this->F2->getOS().SetUnbuffered(); + } + size_t PreferredBufferSize; + std::unique_ptr F1; + std::unique_ptr F2; + }; + struct MirroringOutputBackend : public ProxyOutputBackend1, + public ProxyOutputBackend2 { + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + std::unique_ptr File1; + std::unique_ptr File2; + if (Error E = + ProxyOutputBackend1::createFileImpl(Path, Config).moveInto(File1)) + return std::move(E); + if (Error E = + ProxyOutputBackend2::createFileImpl(Path, Config).moveInto(File2)) + return joinErrors(std::move(E), File1->discard()); + + // Skip the extra indirection if one of these is a null output. + if (isa(*File1)) { + consumeError(File1->discard()); + return std::move(File2); + } + if (isa(*File2)) { + consumeError(File2->discard()); + return std::move(File1); + } + return std::make_unique(std::move(File1), + std::move(File2)); + } + + IntrusiveRefCntPtr cloneImpl() const override { + return IntrusiveRefCntPtr( + makeIntrusiveRefCnt( + ProxyOutputBackend1::getUnderlyingBackend().clone(), + ProxyOutputBackend2::getUnderlyingBackend().clone())); + } + void Retain() const { ProxyOutputBackend1::Retain(); } + void Release() const { ProxyOutputBackend1::Release(); } + + MirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) + : ProxyOutputBackend1(std::move(Backend1)), + ProxyOutputBackend2(std::move(Backend2)) {} + }; + + assert(Backend1 && "Expected actual backend"); + assert(Backend2 && "Expected actual backend"); + return IntrusiveRefCntPtr( + makeIntrusiveRefCnt(std::move(Backend1), + std::move(Backend2))); +} + +static OutputConfig +applySettings(std::optional &&Config, + const OnDiskOutputBackend::OutputSettings &Settings) { + if (!Config) + Config = Settings.DefaultConfig; + if (Settings.DisableTemporaries) + Config->setNoAtomicWrite(); + if (Settings.DisableRemoveOnSignal) + Config->setNoDiscardOnSignal(); + return *Config; +} + +namespace { +class OnDiskOutputFile final : public OutputFileImpl { +public: + Error keep() override; + Error discard() override; + raw_pwrite_stream &getOS() override { + assert(FileOS && "Expected valid file"); + if (BufferOS) + return *BufferOS; + return *FileOS; + } + + /// Attempt to open a temporary file for \p OutputPath. + /// + /// This tries to open a uniquely-named temporary file for \p OutputPath, + /// possibly also creating any missing directories if \a + /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a + /// Config. + /// + /// \post FD and \a TempPath are initialized if this is successful. + Error tryToCreateTemporary(std::optional &FD); + + Error initializeFD(std::optional &FD); + Error initializeStream(); + + OnDiskOutputFile(StringRef OutputPath, std::optional Config, + const OnDiskOutputBackend::OutputSettings &Settings) + : Config(applySettings(std::move(Config), Settings)), + OutputPath(OutputPath.str()) {} + + OutputConfig Config; + const std::string OutputPath; + std::optional TempPath; + std::optional FileOS; + std::optional BufferOS; +}; +} // end namespace + +static Error createDirectoriesOnDemand(StringRef OutputPath, + OutputConfig Config, + llvm::function_ref CreateFile) { + return handleErrors(CreateFile(), [&](std::unique_ptr EC) { + if (EC->convertToErrorCode() != std::errc::no_such_file_or_directory || + Config.getNoImplyCreateDirectories()) + return Error(std::move(EC)); + + StringRef ParentPath = sys::path::parent_path(OutputPath); + if (std::error_code EC = sys::fs::create_directories(ParentPath)) + return make_error(ParentPath, EC); + return CreateFile(); + }); +} + +Error OnDiskOutputFile::tryToCreateTemporary(std::optional &FD) { + // Create a temporary file. + // Insert -%%%%%%%% before the extension (if any), and because some tools + // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build + // artifacts, also append .tmp. + StringRef OutputExtension = sys::path::extension(OutputPath); + SmallString<128> TempPath = + StringRef(OutputPath).drop_back(OutputExtension.size()); + TempPath += "-%%%%%%%%"; + TempPath += OutputExtension; + TempPath += ".tmp"; + + return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error { + int NewFD; + if (std::error_code EC = + sys::fs::createUniqueFile(TempPath, NewFD, TempPath)) + return make_error(TempPath, OutputPath, EC); + + if (Config.getDiscardOnSignal()) + sys::RemoveFileOnSignal(TempPath); + + this->TempPath = TempPath.str().str(); + FD.emplace(NewFD); + return Error::success(); + }); +} + +Error OnDiskOutputFile::initializeFD(std::optional &FD) { + assert(OutputPath != "-" && "Unexpected request for FD of stdout"); + + // Disable temporary file for other non-regular files, and if we get a status + // object, also check if we can write and disable write-through buffers if + // appropriate. + if (Config.getAtomicWrite()) { + sys::fs::file_status Status; + sys::fs::status(OutputPath, Status); + if (sys::fs::exists(Status)) { + if (!sys::fs::is_regular_file(Status)) + Config.setNoAtomicWrite(); + + // Fail now if we can't write to the final destination. + if (!sys::fs::can_write(OutputPath)) + return make_error( + OutputPath, + std::make_error_code(std::errc::operation_not_permitted)); + } + } + + // If (still) using a temporary file, try to create it (and return success if + // that works). + if (Config.getAtomicWrite()) + if (!errorToBool(tryToCreateTemporary(FD))) + return Error::success(); + + // Not using a temporary file. Open the final output file. + return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error { + int NewFD; + sys::fs::OpenFlags OF = sys::fs::OF_None; + if (Config.getTextWithCRLF()) + OF |= sys::fs::OF_TextWithCRLF; + else if (Config.getText()) + OF |= sys::fs::OF_Text; + if (std::error_code EC = sys::fs::openFileForWrite( + OutputPath, NewFD, sys::fs::CD_CreateAlways, OF)) + return convertToOutputError(OutputPath, EC); + FD.emplace(NewFD); + + if (Config.getDiscardOnSignal()) + sys::RemoveFileOnSignal(OutputPath); + return Error::success(); + }); +} + +Error OnDiskOutputFile::initializeStream() { + // Open the file stream. + if (OutputPath == "-") { + std::error_code EC; + FileOS.emplace(OutputPath, EC); + if (EC) + return make_error(OutputPath, EC); + } else { + std::optional FD; + if (Error E = initializeFD(FD)) + return E; + FileOS.emplace(*FD, /*shouldClose=*/true); + } + + // Buffer the stream if necessary. + if (!FileOS->supportsSeeking() && !Config.getText()) + BufferOS.emplace(*FileOS); + + return Error::success(); +} + +Error OnDiskOutputFile::keep() { + // Destroy the streams to flush them. + BufferOS.reset(); + FileOS.reset(); + + // Close the file descriptor and remove crash cleanup before exit. + auto RemoveDiscardOnSignal = make_scope_exit([&]() { + if (Config.getDiscardOnSignal()) + sys::DontRemoveFileOnSignal(TempPath ? *TempPath : OutputPath); + }); + + if (!TempPath) + return Error::success(); + + // Move temporary to the final output path and remove it if that fails. + std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath); + if (!RenameEC) + return Error::success(); + + // Copy the output to see if makes any difference. If this path is used, + // investigate why we need to copy. + RenameEC = sys::fs::copy_file(*TempPath, OutputPath); + (void)sys::fs::remove(*TempPath); + return make_error(*TempPath, OutputPath, RenameEC); +} + +Error OnDiskOutputFile::discard() { + // Destroy the streams to flush them. + BufferOS.reset(); + FileOS.reset(); + + // Nothing on the filesystem to remove for stdout. + if (OutputPath == "-") + return Error::success(); + + auto discardPath = [&](StringRef Path) { + std::error_code EC = sys::fs::remove(Path); + sys::DontRemoveFileOnSignal(Path); + return EC; + }; + + // Clean up the file that's in-progress. + if (!TempPath) + return convertToOutputError(OutputPath, discardPath(OutputPath)); + return convertToTempFileOutputError(*TempPath, OutputPath, + discardPath(*TempPath)); +} + +Error OnDiskOutputBackend::makeAbsolute(SmallVectorImpl &Path) const { + return convertToOutputError(StringRef(Path.data(), Path.size()), + sys::fs::make_absolute(Path)); +} + +Expected> +OnDiskOutputBackend::createFileImpl(StringRef Path, + std::optional Config) { + SmallString<256> AbsPath; + if (Path != "-") { + AbsPath = Path; + if (Error E = makeAbsolute(AbsPath)) + return std::move(E); + Path = AbsPath; + } + + auto File = std::make_unique(Path, Config, Settings); + if (Error E = File->initializeStream()) + return std::move(E); + + return std::move(File); +} diff --git a/llvm/lib/Support/VirtualOutputConfig.cpp b/llvm/lib/Support/VirtualOutputConfig.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputConfig.cpp @@ -0,0 +1,48 @@ +//===- VirtualOutputConfig.cpp - Virtual output configuration -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::vfs; + +OutputConfig &OutputConfig::setOpenFlags(const sys::fs::OpenFlags &Flags) { + // Ignore CRLF on its own as invalid. + using namespace llvm::sys::fs; + return Flags & OF_Text ? setText().setCRLF(Flags & OF_CRLF) : setBinary(); +} + +void OutputConfig::print(raw_ostream &OS) const { + OS << "{"; + bool IsFirst = true; + auto printFlag = [&](StringRef FlagName, bool Value) { + if (IsFirst) + IsFirst = false; + else + OS << ","; + if (!Value) + OS << "No"; + OS << FlagName; + }; + +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \ + if (get##NAME() != DEFAULT) \ + printFlag(#NAME, get##NAME()); +#include "llvm/Support/VirtualOutputConfig.def" + OS << "}"; +} + +LLVM_DUMP_METHOD void OutputConfig::dump() const { print(dbgs()); } + +raw_ostream &llvm::operator<<(raw_ostream &OS, OutputConfig Config) { + Config.print(OS); + return OS; +} diff --git a/llvm/lib/Support/VirtualOutputError.cpp b/llvm/lib/Support/VirtualOutputError.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputError.cpp @@ -0,0 +1,53 @@ +//===- VirtualOutputError.cpp - Errors for output virtualization ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputError.h" + +using namespace llvm; +using namespace llvm::vfs; + +void OutputError::anchor() {} +void OutputConfigError::anchor() {} +void TempFileOutputError::anchor() {} + +char OutputError::ID = 0; +char OutputConfigError::ID = 0; +char TempFileOutputError::ID = 0; + +namespace { +class OutputErrorCategory : public std::error_category { +public: + const char *name() const noexcept override; + std::string message(int EV) const override; +}; +} // end namespace + +const std::error_category &vfs::output_category() { + static OutputErrorCategory ErrorCategory; + return ErrorCategory; +} + +const char *OutputErrorCategory::name() const noexcept { + return "llvm.vfs.output"; +} + +std::string OutputErrorCategory::message(int EV) const { + OutputErrorCode E = static_cast(EV); + switch (E) { + case OutputErrorCode::invalid_config: + return "invalid config"; + case OutputErrorCode::not_closed: + return "output not closed"; + case OutputErrorCode::already_closed: + return "output already closed"; + case OutputErrorCode::has_open_proxy: + return "output has open proxy"; + } + llvm_unreachable( + "An enumerator of OutputErrorCode does not have a message defined."); +} diff --git a/llvm/lib/Support/VirtualOutputFile.cpp b/llvm/lib/Support/VirtualOutputFile.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputFile.cpp @@ -0,0 +1,106 @@ +//===- VirtualOutputFile.cpp - Output file virtualization -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputFile.h" +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/Support/VirtualOutputError.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/raw_ostream_proxy.h" + +using namespace llvm; +using namespace llvm::vfs; + +char OutputFileImpl::ID = 0; +char NullOutputFileImpl::ID = 0; + +void OutputFileImpl::anchor() {} +void NullOutputFileImpl::anchor() {} + +class OutputFile::TrackedProxy : public raw_pwrite_stream_proxy { +public: + void resetProxy() { + TrackingPointer = nullptr; + resetProxiedOS(); + } + + explicit TrackedProxy(TrackedProxy *&TrackingPointer, raw_pwrite_stream &OS) + : raw_pwrite_stream_proxy(OS), TrackingPointer(TrackingPointer) { + assert(!TrackingPointer && "Expected to add a proxy"); + TrackingPointer = this; + } + + ~TrackedProxy() override { resetProxy(); } + + TrackedProxy *&TrackingPointer; +}; + +Expected> OutputFile::createProxy() { + if (OpenProxy) + return make_error(getPath(), OutputErrorCode::has_open_proxy); + + return std::make_unique(OpenProxy, getOS()); +} + +Error OutputFile::keep() { + // Catch double-closing logic bugs. + if (LLVM_UNLIKELY(!Impl)) + report_fatal_error( + make_error(getPath(), OutputErrorCode::already_closed)); + + // Report a fatal error if there's an open proxy and the file is being kept. + // This is safer than relying on clients to remember to flush(). Also call + // OutputFile::discard() to give the backend a chance to clean up any + // side effects (such as temporaries). + if (LLVM_UNLIKELY(OpenProxy)) + report_fatal_error(joinErrors( + make_error(getPath(), OutputErrorCode::has_open_proxy), + discard())); + + Error E = Impl->keep(); + Impl = nullptr; + DiscardOnDestroyHandler = nullptr; + return E; +} + +Error OutputFile::discard() { + // Catch double-closing logic bugs. + if (LLVM_UNLIKELY(!Impl)) + report_fatal_error( + make_error(getPath(), OutputErrorCode::already_closed)); + + // Be lenient about open proxies since client teardown paths won't + // necessarily clean up in the right order. Reset the proxy to flush any + // current content; if there is another write, there should be quick crash on + // null dereference. + if (OpenProxy) + OpenProxy->resetProxy(); + + Error E = Impl->discard(); + Impl = nullptr; + DiscardOnDestroyHandler = nullptr; + return E; +} + +void OutputFile::destroy() { + if (!Impl) + return; + + // Clean up the file. Move the discard handler into a local since discard + // will reset it. + auto DiscardHandler = std::move(DiscardOnDestroyHandler); + Error E = discard(); + assert(!Impl && "Expected discard to destroy Impl"); + + // If there's no handler, report a fatal error. + if (LLVM_UNLIKELY(!DiscardHandler)) + llvm::report_fatal_error(joinErrors( + make_error(getPath(), OutputErrorCode::not_closed), + std::move(E))); + else if (E) + DiscardHandler(std::move(E)); +} diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt --- a/llvm/unittests/Support/CMakeLists.txt +++ b/llvm/unittests/Support/CMakeLists.txt @@ -92,6 +92,10 @@ UnicodeTest.cpp VersionTupleTest.cpp VirtualFileSystemTest.cpp + VirtualOutputBackendTest.cpp + VirtualOutputBackendsTest.cpp + VirtualOutputConfigTest.cpp + VirtualOutputFileTest.cpp WithColorTest.cpp YAMLIOTest.cpp YAMLParserTest.cpp diff --git a/llvm/unittests/Support/VirtualOutputBackendTest.cpp b/llvm/unittests/Support/VirtualOutputBackendTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputBackendTest.cpp @@ -0,0 +1,147 @@ +//===- VirtualOutputBackendTest.cpp - Tests for vfs::OutputBackend --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +struct MockOutputBackendData { + int Cloned = 0; + int FilesCreated = 0; + std::optional LastConfig; + unique_function FileCreator; +}; + +struct MockOutputBackend final : public OutputBackend { + struct MockFile final : public OutputFileImpl { + Error keep() override { return Error::success(); } + Error discard() override { return Error::success(); } + raw_pwrite_stream &getOS() override { return OS; } + raw_null_ostream OS; + }; + + IntrusiveRefCntPtr cloneImpl() const override { + ++Data.Cloned; + return const_cast(this); + } + + Expected> + createFileImpl(StringRef, std::optional Config) override { + ++Data.FilesCreated; + Data.LastConfig = Config; + if (Data.FileCreator) + return Data.FileCreator(); + return std::make_unique(); + } + + Expected + createAutoDiscardFile(const Twine &OutputPath, + std::optional Config = std::nullopt) { + return consumeDiscardOnDestroy(createFile(OutputPath, Config)); + } + + MockOutputBackend(MockOutputBackendData &Data) : Data(Data) {} + MockOutputBackendData &Data; +}; + +static IntrusiveRefCntPtr +createMockBackend(MockOutputBackendData &Data) { + return makeIntrusiveRefCnt(Data); +} + +static Error createCustomError() { + return createStringError(inconvertibleErrorCode(), "custom error"); +} + +TEST(VirtualOutputBackendTest, construct) { + MockOutputBackendData Data; + auto B = createMockBackend(Data); + EXPECT_EQ(0, Data.Cloned); + EXPECT_EQ(0, Data.FilesCreated); +} + +TEST(VirtualOutputBackendTest, clone) { + MockOutputBackendData Data; + auto Backend = createMockBackend(Data); + auto Clone = Backend->clone(); + EXPECT_EQ(1, Data.Cloned); + + // Confirm the clone matches what the mock's cloneImpl() does. + EXPECT_EQ(Backend.get(), Clone.get()); + + // Make another clone. + Backend->clone(); + EXPECT_EQ(2, Data.Cloned); +} + +TEST(VirtualOutputBackendTest, createFile) { + MockOutputBackendData Data; + auto Backend = createMockBackend(Data); + + StringRef FilePath = "dir/file"; + OutputFile F; + EXPECT_THAT_ERROR(Backend->createFile(Twine(FilePath)).moveInto(F), + Succeeded()); + EXPECT_EQ(1, Data.FilesCreated); + EXPECT_EQ(FilePath, F.getPath()); + EXPECT_EQ(std::nullopt, Data.LastConfig); + + // Confirm OutputBackend has not installed a discard handler. +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(F = OutputFile(), "output not closed"); +#endif + consumeError(F.discard()); + + // Create more files and specify configs. + for (OutputConfig Config : { + OutputConfig(), + OutputConfig().setNoAtomicWrite().setDiscardOnSignal(), + OutputConfig().setAtomicWrite().setNoDiscardOnSignal(), + OutputConfig().setText(), + OutputConfig().setTextWithCRLF(), + }) { + int CreatedAlready = Data.FilesCreated; + EXPECT_THAT_ERROR( + Backend->createAutoDiscardFile(Twine(FilePath), Config).takeError(), + Succeeded()); + EXPECT_EQ(Config, Data.LastConfig); + EXPECT_EQ(1 + CreatedAlready, Data.FilesCreated); + } +} + +TEST(VirtualOutputBackendTest, createFileInvalidConfigCRLF) { + MockOutputBackendData Data; + auto Backend = createMockBackend(Data); + + // Check that invalid configs don't make it to the backend. + EXPECT_THAT_ERROR( + Backend + ->createAutoDiscardFile(Twine("dir/file"), OutputConfig().setCRLF()) + .takeError(), + FailedWithMessage("dir/file: invalid config: {CRLF}")); + EXPECT_EQ(0, Data.FilesCreated); +} + +TEST(VirtualOutputBackendTest, createFileError) { + MockOutputBackendData Data; + Data.FileCreator = createCustomError; + auto Backend = createMockBackend(Data); + + // Check that invalid configs don't make it to the backend. + EXPECT_THAT_ERROR( + Backend->createAutoDiscardFile(Twine("dir/file")).takeError(), + FailedWithMessage("custom error")); + EXPECT_EQ(1, Data.FilesCreated); +} + +} // end namespace diff --git a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp @@ -0,0 +1,770 @@ +//===- VirtualOutputBackendsTest.cpp - Tests for vfs::OutputBackend impls -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +class OutputBackendProvider { +public: + virtual bool rejectsMissingDirectories() = 0; + + virtual IntrusiveRefCntPtr createBackend() = 0; + virtual std::string getFilePathToCreate() = 0; + virtual std::string getFilePathToCreateUnder(StringRef Parent1, + StringRef Parent2 = "") = 0; + virtual Error checkCreated(StringRef FilePath, + OutputConfig Config = OutputConfig()) = 0; + virtual Error checkWrote(StringRef FilePath, StringRef Data) = 0; + virtual Error checkFlushed(StringRef FilePath, StringRef Data) = 0; + virtual Error checkKept(StringRef FilePath, StringRef Data) = 0; + virtual Error checkDiscarded(StringRef FilePath) = 0; + + virtual ~OutputBackendProvider() = default; + + struct Generator { + std::string Name; + std::function()> Generate; + + std::unique_ptr operator()() const { + return Generate(); + } + }; +}; + +struct BackendTest + : public ::testing::TestWithParam { + std::unique_ptr Provider; + + void SetUp() override { Provider = GetParam()(); } + void TearDown() override { Provider = nullptr; } + + IntrusiveRefCntPtr createBackend() { + return Provider->createBackend(); + } +}; + +TEST_P(BackendTest, Discard) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.discard(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, DiscardNoAtomicWrite) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setNoAtomicWrite(); + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O), + Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.discard(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, Keep) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + ASSERT_TRUE(O.isOpen()); + + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, KeepFlush) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + O.getOS().flush(); + EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepFlushProxy) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + { + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(O.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + Proxy->flush(); + EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded()); + } + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepEmpty) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, ""), Succeeded()); +} + +TEST_P(BackendTest, KeepMissingDirectory) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreateUnder("missing"); + StringRef Data = "some data"; + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepMissingDirectoryNested) { + auto Backend = createBackend(); + std::string FilePath = + Provider->getFilePathToCreateUnder("missing", "nested"); + StringRef Data = "some data"; + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepNoAtomicWrite) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setNoAtomicWrite(); + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O), + Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded()); + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, KeepNoAtomicWriteMissingDirectory) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setNoAtomicWrite(); + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O), + Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, KeepMissingDirectoryNoImply) { + // Skip this test if the backend doesn't have a concept of missing + // directories. + if (!Provider->rejectsMissingDirectories()) + return; + + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreateUnder("missing"); + std::error_code EC = errorToErrorCode( + consumeDiscardOnDestroy( + Backend->createFile(FilePath, + OutputConfig().setNoImplyCreateDirectories())) + .takeError()); + EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value()); +} + +class NullOutputBackendProvider : public OutputBackendProvider { +public: + bool rejectsMissingDirectories() override { return false; } + + IntrusiveRefCntPtr createBackend() override { + return makeNullOutputBackend(); + } + std::string getFilePathToCreate() override { return "ignored.data"; } + std::string getFilePathToCreateUnder(StringRef Parent1, + StringRef Parent2) override { + SmallString<128> Path; + sys::path::append(Path, Parent1, Parent2, getFilePathToCreate()); + return Path.str().str(); + } + Error checkCreated(StringRef, OutputConfig) override { + return Error::success(); + } + Error checkWrote(StringRef, StringRef) override { return Error::success(); } + Error checkFlushed(StringRef, StringRef) override { return Error::success(); } + Error checkKept(StringRef, StringRef) override { return Error::success(); } + Error checkDiscarded(StringRef) override { return Error::success(); } +}; + +struct OnDiskFile { + const unittest::TempDir &D; + SmallString<128> Path; + StringRef ParentPath; + StringRef Filename; + StringRef Stem; + StringRef Extension; + std::unique_ptr LastBuffer; + + OnDiskFile(const unittest::TempDir &D, const Twine &InputPath) : D(D) { + if (sys::path::is_absolute(InputPath)) + InputPath.toVector(Path); + else + sys::path::append(Path, D.path(), InputPath); + ParentPath = sys::path::parent_path(Path); + Filename = sys::path::filename(Path); + Stem = sys::path::stem(Filename); + Extension = sys::path::extension(Filename); + } + + std::optional findTemp() const; + + std::optional getCurrentUniqueID(); + + bool hasUniqueID(sys::fs::UniqueID ID) { + auto CurrentID = getCurrentUniqueID(); + if (!CurrentID) + return false; + return *CurrentID == ID; + } + + std::optional getCurrentContent() { + auto OnDiskOrErr = MemoryBuffer::getFile(Path); + if (!OnDiskOrErr) + return std::nullopt; + LastBuffer = std::move(*OnDiskOrErr); + return LastBuffer->getBuffer(); + } + + bool equalsCurrentContent(StringRef Data) { + auto CurrentContent = getCurrentContent(); + if (!CurrentContent) + return false; + return *CurrentContent == Data; + } + + bool equalsCurrentContent(std::nullopt_t) { + return getCurrentContent() == std::nullopt; + } +}; + +class OnDiskOutputBackendProvider : public OutputBackendProvider { +public: + bool rejectsMissingDirectories() override { return true; } + + std::optional D; + + IntrusiveRefCntPtr createBackend() override { + auto Backend = makeIntrusiveRefCnt(); + Backend->Settings = Settings; + return Backend; + } + void init() { + if (!D) + D.emplace("OutputBackendTest.d", /*Unique=*/true); + } + std::string getFilePathToCreate() override { + init(); + return OnDiskFile(*D, "file.data").Path.str().str(); + } + std::string getFilePathToCreateUnder(StringRef Parent1, + StringRef Parent2) override { + init(); + SmallString<128> Path; + sys::path::append(Path, D->path(), Parent1, Parent2, getFilePathToCreate()); + return Path.str().str(); + } + + Error checkCreated(StringRef FilePath, OutputConfig Config) override; + Error checkWrote(StringRef FilePath, StringRef Data) override; + Error checkFlushed(StringRef FilePath, StringRef Data) override; + Error checkKept(StringRef FilePath, StringRef Data) override; + Error checkDiscarded(StringRef FilePath) override; + + struct FileInfo { + OutputConfig Config; + std::optional F; + std::optional Temp; + std::optional UID; + std::optional TempUID; + }; + Error checkOpen(FileInfo &Info); + bool shouldUseTemporaries(const FileInfo &Info) const; + + OnDiskOutputBackendProvider() = default; + explicit OnDiskOutputBackendProvider( + const OnDiskOutputBackend::OutputSettings &Settings) + : Settings(Settings) {} + OnDiskOutputBackend::OutputSettings Settings; + + StringMap Files; + Error lookupFileInfo(StringRef FilePath, FileInfo *&Info); +}; + +bool OnDiskOutputBackendProvider::shouldUseTemporaries( + const FileInfo &Info) const { + return Info.Config.getAtomicWrite() && !Settings.DisableTemporaries; +} + +struct ProviderGeneratorList { + std::vector Generators; + ProviderGeneratorList( + std::initializer_list IL) + : Generators(IL) {} + + std::string operator()( + const ::testing::TestParamInfo &Info) { + return Info.param.Name; + } +}; + +ProviderGeneratorList BackendGenerators = { + {"Null", []() { return std::make_unique(); }}, + {"OnDisk", + []() { return std::make_unique(); }}, + {"OnDisk_DisableRemoveOnSignal", + []() { + OnDiskOutputBackend::OutputSettings Settings; + Settings.DisableRemoveOnSignal = true; + return std::make_unique(Settings); + }}, + {"OnDisk_DisableTemporaries", + []() { + OnDiskOutputBackend::OutputSettings Settings; + Settings.DisableTemporaries = true; + return std::make_unique(Settings); + }}, +}; + +INSTANTIATE_TEST_SUITE_P(VirtualOutput, BackendTest, + ::testing::ValuesIn(BackendGenerators.Generators), + BackendGenerators); + +std::optional OnDiskFile::getCurrentUniqueID() { + sys::fs::file_status Status; + sys::fs::status(Path, Status, /*follow=*/false); + if (!sys::fs::is_regular_file(Status)) + return std::nullopt; + return Status.getUniqueID(); +} + +std::optional OnDiskFile::findTemp() const { + std::error_code EC; + for (sys::fs::directory_iterator I(ParentPath, EC), E; !EC && I != E; + I.increment(EC)) { + StringRef TempPath = I->path(); + if (!TempPath.startswith(D.path())) + continue; + + // Look for "-*..tmp". + if (sys::path::extension(TempPath) != ".tmp") + continue; + + // Drop the ".tmp" and check the extension and stem. + StringRef TempStem = sys::path::stem(TempPath); + if (sys::path::extension(TempStem) != Extension) + continue; + StringRef OriginalStem = sys::path::stem(TempStem); + if (!OriginalStem.startswith(Stem)) + continue; + if (!OriginalStem.drop_front(Stem.size()).startswith("-")) + continue; + + // Found it. + return OnDiskFile(D, TempPath.drop_front(D.path().size() + 1)); + } + return std::nullopt; +} + +Error OnDiskOutputBackendProvider::lookupFileInfo(StringRef FilePath, + FileInfo *&Info) { + auto I = Files.find(FilePath); + if (Files.find(FilePath) == Files.end()) + return createStringError(inconvertibleErrorCode(), + "Missing call to checkCreated()"); + Info = &I->second; + assert(Info->F && "Expected OnDiskFile to be initialized"); + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkOpen(FileInfo &Info) { + // Collect info about filesystem state. + assert(Info.F); + std::optional UID = Info.F->getCurrentUniqueID(); + std::optional Temp = Info.F->findTemp(); + std::optional TempUID; + if (Temp) + TempUID = Temp->getCurrentUniqueID(); + + // Check if it's correct. + if (shouldUseTemporaries(Info)) { + if (!Temp) + return createStringError(inconvertibleErrorCode(), + "Missing temporary file"); + if (!TempUID) + return createStringError(inconvertibleErrorCode(), + "Missing UID for temporary"); + if (UID) + return createStringError( + inconvertibleErrorCode(), + "Unexpected final UID when temporaries should be used"); + + // Check previous data. + if (Info.Temp) + if (Temp->Path != Info.Temp->Path) + return createStringError(inconvertibleErrorCode(), + "Temporary path changed"); + if (Info.TempUID) + if (*TempUID != *Info.TempUID) + return createStringError(inconvertibleErrorCode(), + "Temporary UID changed"); + } else { + if (Temp) + return createStringError(inconvertibleErrorCode(), + "Unexpected temporary file"); + if (!UID) + return createStringError(inconvertibleErrorCode(), + "Missing UID for temporary"); + + // Check previous data. + if (Info.UID) + if (*UID != *Info.UID) + return createStringError(inconvertibleErrorCode(), "UID changed"); + } + + Info.UID = UID; + if (Temp) + Info.Temp.emplace(*D, Temp->Path); + else + Info.Temp.reset(); + Info.TempUID = TempUID; + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkCreated(StringRef FilePath, + OutputConfig Config) { + auto &Info = Files[FilePath]; + if (Info.F) { + assert(OnDiskFile(*D, FilePath).Path == Info.F->Path); + Info.UID = std::nullopt; + Info.Temp.reset(); + Info.TempUID = std::nullopt; + } else { + Info.F.emplace(*D, FilePath); + } + Info.Config = Config; + return checkOpen(Info); +} + +Error OnDiskOutputBackendProvider::checkWrote(StringRef FilePath, + StringRef Data) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + return checkOpen(*Info); +} + +Error OnDiskOutputBackendProvider::checkFlushed(StringRef FilePath, + StringRef Data) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + if (Error E = checkOpen(*Info)) + return E; + + OnDiskFile &F = shouldUseTemporaries(*Info) ? *Info->Temp : *Info->F; + if (!F.equalsCurrentContent(Data)) + return createStringError(inconvertibleErrorCode(), "content not flushed"); + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkKept(StringRef FilePath, + StringRef Data) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + + sys::fs::UniqueID UID = + shouldUseTemporaries(*Info) ? *Info->TempUID : *Info->UID; + if (!Info->F->hasUniqueID(UID)) + return createStringError(inconvertibleErrorCode(), + "File not created by keep or changed UID"); + + if (std::optional Temp = Info->F->findTemp()) + return createStringError(inconvertibleErrorCode(), + "Temporary not removed by keep"); + + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkDiscarded(StringRef FilePath) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + + if (std::optional UID = Info->F->getCurrentUniqueID()) + return createStringError(inconvertibleErrorCode(), + "File not removed by discard"); + + if (std::optional Temp = Info->F->findTemp()) + return createStringError(inconvertibleErrorCode(), + "Temporary not removed by discard"); + + return Error::success(); +} + +TEST(VirtualOutputBackendAdaptors, makeFilteringOutputBackend) { + bool ShouldCreate = false; + auto Backend = makeFilteringOutputBackend( + makeIntrusiveRefCnt(), + [&ShouldCreate](StringRef, std::optional) { + return ShouldCreate; + }); + + int Count = 0; + unittest::TempDir D("FilteringOutputBackendTest.d", /*Unique=*/true); + for (bool ShouldCreateVal : {false, true, true, false}) { + ShouldCreate = ShouldCreateVal; + OnDiskFile OnDisk(D, "file." + Twine(Count++) + "." + Twine(ShouldCreate)); + OutputFile Output; + ASSERT_THAT_ERROR(consumeDiscardOnDestroy(Backend->createFile(OnDisk.Path)) + .moveInto(Output), + Succeeded()); + EXPECT_NE(ShouldCreate, Output.isNull()); + Output << "content"; + EXPECT_THAT_ERROR(Output.keep(), Succeeded()); + + if (ShouldCreate) { + EXPECT_EQ(StringRef("content"), OnDisk.getCurrentContent()); + } else { + EXPECT_FALSE(OnDisk.getCurrentUniqueID()); + } + } + SmallString<128> Path; +} + +class AbsolutePathBackend : public ProxyOutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + llvm_unreachable("unimplemented"); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + assert(!sys::path::is_absolute(Path) && + "Expected tests to pass all relative paths"); + SmallString<256> AbsPath; + sys::path::append(AbsPath, CWD, Path); + return ProxyOutputBackend::createFileImpl(AbsPath, Config); + } + +public: + AbsolutePathBackend(const Twine &CWD, + IntrusiveRefCntPtr Backend) + : ProxyOutputBackend(std::move(Backend)), CWD(CWD.str()) { + assert(sys::path::is_absolute(this->CWD) && + "Expected tests to pass a relative path"); + } + +private: + std::string CWD; +}; + +TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackend) { + unittest::TempDir D1("MirroringOutputBackendTest.1.d", /*Unique=*/true); + unittest::TempDir D2("MirroringOutputBackendTest.2.d", /*Unique=*/true); + + IntrusiveRefCntPtr Backend; + { + auto OnDisk = makeIntrusiveRefCnt(); + Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(D1.path(), OnDisk), + makeIntrusiveRefCnt(D2.path(), OnDisk)); + } + + OnDiskFile OnDisk1(D1, "file"); + OnDiskFile OnDisk2(D2, "file"); + OutputFile Output; + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Backend->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(OnDisk1.findTemp()); + EXPECT_TRUE(OnDisk2.findTemp()); + + Output << "content"; + Output.getOS().pwrite("ON", /*Size=*/2, /*Offset=*/1); + EXPECT_THAT_ERROR(Output.keep(), Succeeded()); + EXPECT_EQ(StringRef("cONtent"), OnDisk1.getCurrentContent()); + EXPECT_EQ(StringRef("cONtent"), OnDisk2.getCurrentContent()); + EXPECT_NE(OnDisk1.getCurrentUniqueID(), OnDisk2.getCurrentUniqueID()); +} + +/// Behaves like NullOutputFileImpl, but doesn't match the RTTI (so OutputFile +/// cannot tell). +class LikeNullOutputFile final : public OutputFileImpl { + Error keep() final { return Error::success(); } + Error discard() final { return Error::success(); } + raw_pwrite_stream &getOS() final { return OS; } + +public: + LikeNullOutputFile(raw_null_ostream &OS) : OS(OS) {} + raw_null_ostream &OS; +}; +class LikeNullOutputBackend final : public OutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + llvm_unreachable("not implemented"); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + return std::make_unique(OS); + } + +public: + raw_null_ostream OS; +}; + +TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendNull) { + // Check that null outputs are skipped by seeing that LikeNull->OS is passed + // through directly (without a mirroring proxy stream) to Output. + auto LikeNull = makeIntrusiveRefCnt(); + auto Null1 = makeNullOutputBackend(); + auto Mirror = makeMirroringOutputBackend(Null1, LikeNull); + OutputFile Output; + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(!Output.isNull()); + EXPECT_EQ(&Output.getOS(), &LikeNull->OS); + + // Check the other direction. + Mirror = makeMirroringOutputBackend(LikeNull, Null1); + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(!Output.isNull()); + EXPECT_EQ(&Output.getOS(), &LikeNull->OS); + + // Same null backend, twice. + Mirror = makeMirroringOutputBackend(Null1, Null1); + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(Output.isNull()); + + // Two null backends. + auto Null2 = makeNullOutputBackend(); + Mirror = makeMirroringOutputBackend(Null1, Null2); + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(Output.isNull()); +} + +class StringErrorBackend final : public OutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + llvm_unreachable("not implemented"); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + return createStringError(inconvertibleErrorCode(), Msg); + } + +public: + StringErrorBackend(const Twine &Msg) : Msg(Msg.str()) {} + std::string Msg; +}; + +TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) { + auto Error1 = makeIntrusiveRefCnt("error-backend-1"); + auto Null = makeNullOutputBackend(); + + auto Mirror = makeMirroringOutputBackend(Null, Error1); + EXPECT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(), + FailedWithMessage(Error1->Msg)); + + Mirror = makeMirroringOutputBackend(Error1, Null); + EXPECT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(), + FailedWithMessage(Error1->Msg)); + + auto Error2 = makeIntrusiveRefCnt("error-backend-2"); + Mirror = makeMirroringOutputBackend(Error1, Error2); + EXPECT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(), + FailedWithMessage(Error1->Msg)); +} + +} // end namespace diff --git a/llvm/unittests/Support/VirtualOutputConfigTest.cpp b/llvm/unittests/Support/VirtualOutputConfigTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputConfigTest.cpp @@ -0,0 +1,147 @@ +//===- VirtualOutputConfigTest.cpp - vfs::OutputConfig tests --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/FileSystem.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +TEST(VirtualOutputConfigTest, construct) { + // Test defaults. + EXPECT_FALSE(OutputConfig().getText()); + EXPECT_FALSE(OutputConfig().getCRLF()); + EXPECT_TRUE(OutputConfig().getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig().getAtomicWrite()); + EXPECT_TRUE(OutputConfig().getImplyCreateDirectories()); + + // Test inverted defaults. + EXPECT_TRUE(OutputConfig().getNoText()); + EXPECT_TRUE(OutputConfig().getNoCRLF()); + EXPECT_FALSE(OutputConfig().getNoDiscardOnSignal()); + EXPECT_FALSE(OutputConfig().getNoAtomicWrite()); + EXPECT_FALSE(OutputConfig().getNoImplyCreateDirectories()); +} + +TEST(VirtualOutputConfigTest, set) { + // Check a flag that defaults to false. Try both 'get's, all three 'set's, + // and turning back off after turning it on. + ASSERT_TRUE(OutputConfig().getNoText()); + EXPECT_TRUE(OutputConfig().setText().getText()); + EXPECT_FALSE(OutputConfig().setText().getNoText()); + EXPECT_TRUE(OutputConfig().setText(true).getText()); + EXPECT_FALSE(OutputConfig().setText().setNoText().getText()); + EXPECT_FALSE(OutputConfig().setText().setText(false).getText()); + + // Check a flag that defaults to true. Try both 'get's, all three 'set's, and + // turning back on after turning it off. + ASSERT_TRUE(OutputConfig().getDiscardOnSignal()); + EXPECT_FALSE(OutputConfig().setNoDiscardOnSignal().getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig().setNoDiscardOnSignal().getNoDiscardOnSignal()); + EXPECT_FALSE(OutputConfig().setDiscardOnSignal(false).getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig() + .setNoDiscardOnSignal() + .setDiscardOnSignal() + .getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig() + .setNoDiscardOnSignal() + .setDiscardOnSignal(true) + .getDiscardOnSignal()); + + // Set multiple flags. + OutputConfig Config; + Config.setText().setNoDiscardOnSignal().setNoImplyCreateDirectories(); + EXPECT_TRUE(Config.getText()); + EXPECT_TRUE(Config.getNoDiscardOnSignal()); + EXPECT_TRUE(Config.getNoImplyCreateDirectories()); +} + +TEST(VirtualOutputConfigTest, equals) { + EXPECT_TRUE(OutputConfig() == OutputConfig()); + EXPECT_FALSE(OutputConfig() != OutputConfig()); + EXPECT_EQ(OutputConfig().setAtomicWrite(), OutputConfig().setAtomicWrite()); + EXPECT_NE(OutputConfig().setAtomicWrite(), OutputConfig().setNoAtomicWrite()); +} + +static std::string toString(OutputConfig Config) { + std::string Printed; + raw_string_ostream OS(Printed); + Config.print(OS); + return Printed; +} + +TEST(VirtualOutputConfigTest, print) { + EXPECT_EQ("{}", toString(OutputConfig())); + EXPECT_EQ("{Text}", toString(OutputConfig().setText())); + EXPECT_EQ("{Text,NoDiscardOnSignal}", + toString(OutputConfig().setText().setNoDiscardOnSignal())); + EXPECT_EQ("{Text,NoDiscardOnSignal}", + toString(OutputConfig().setNoDiscardOnSignal().setText())); +} + +TEST(VirtualOutputConfigTest, BinaryAndTextWithCRLF) { + // Test defaults. + EXPECT_TRUE(OutputConfig().getBinary()); + EXPECT_FALSE(OutputConfig().getTextWithCRLF()); + EXPECT_FALSE(OutputConfig().getText()); + EXPECT_FALSE(OutputConfig().getCRLF()); + + // Test setting. + EXPECT_TRUE(OutputConfig().setTextWithCRLF().getTextWithCRLF()); + EXPECT_TRUE(OutputConfig().setTextWithCRLF().getText()); + EXPECT_TRUE(OutputConfig().setTextWithCRLF().getCRLF()); + EXPECT_TRUE(OutputConfig().setText().setCRLF().getTextWithCRLF()); + EXPECT_FALSE(OutputConfig().setText().getBinary()); + EXPECT_FALSE(OutputConfig().setTextWithCRLF().getBinary()); + EXPECT_FALSE(OutputConfig().setTextWithCRLF().setBinary().getText()); + EXPECT_FALSE(OutputConfig().setTextWithCRLF().setBinary().getCRLF()); + + // Test setTextWithCRLF(bool). + EXPECT_TRUE(OutputConfig().setBinary().setTextWithCRLF(true).getText()); + EXPECT_TRUE(OutputConfig().setBinary().setTextWithCRLF(true).getCRLF()); + EXPECT_TRUE( + OutputConfig().setTextWithCRLF().setTextWithCRLF(false).getBinary()); + + // Test printing. + EXPECT_EQ("{Text,CRLF}", toString(OutputConfig().setTextWithCRLF())); +} + +TEST(VirtualOutputConfigTest, OpenFlags) { + using namespace llvm::sys::fs; + + // Confirm the default is binary. + ASSERT_EQ(OutputConfig().setBinary(), OutputConfig()); + + // Most flags are not supported / have no effect. + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_None)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_Append)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_Delete)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_ChildInherit)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_UpdateAtime)); + + // Check setting OF_Text and OF_CRLF. + for (OutputConfig Init : { + OutputConfig(), + OutputConfig().setText(), + OutputConfig().setTextWithCRLF(), + + // Should be overridden despite being invalid. + OutputConfig().setCRLF(), + }) { + EXPECT_EQ(OutputConfig(), Init.setOpenFlags(OF_None)); + EXPECT_EQ(OutputConfig(), Init.setOpenFlags(OF_CRLF)); + EXPECT_EQ(OutputConfig().setText(), Init.setOpenFlags(OF_Text)); + EXPECT_EQ(OutputConfig().setTextWithCRLF(), + Init.setOpenFlags(OF_TextWithCRLF)); + } +} + +} // anonymous namespace diff --git a/llvm/unittests/Support/VirtualOutputFileTest.cpp b/llvm/unittests/Support/VirtualOutputFileTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputFileTest.cpp @@ -0,0 +1,342 @@ +//===- VirtualOutputFileTest.cpp - vfs::OutputFile tests ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputFile.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +struct MockOutputFileData { + int Kept = 0; + int Discarded = 0; + int Handled = 0; + unique_function Keeper; + unique_function Discarder; + + void handler(Error E) { + consumeError(std::move(E)); + ++Handled; + } + unique_function getHandler() { + return [this](Error E) { handler(std::move(E)); }; + } + + SmallString<128> V; + Optional VOS; + raw_pwrite_stream *OS = nullptr; + + MockOutputFileData() : VOS(std::in_place, V), OS(&*VOS) {} + MockOutputFileData(raw_pwrite_stream &OS) : OS(&OS) {} +}; + +struct MockOutputFile final : public OutputFileImpl { + Error keep() override { + ++Data.Kept; + if (Data.Keeper) + return Data.Keeper(); + return Error::success(); + } + + Error discard() override { + ++Data.Discarded; + if (Data.Discarder) + return Data.Discarder(); + return Error::success(); + } + + raw_pwrite_stream &getOS() override { + if (!Data.OS) + report_fatal_error("missing stream in MockOutputFile::getOS"); + return *Data.OS; + } + + MockOutputFile(MockOutputFileData &Data) : Data(Data) {} + MockOutputFileData &Data; +}; + +static std::unique_ptr +createMockOutput(MockOutputFileData &Data) { + return std::make_unique(Data); +} + +static Error createCustomError() { + return createStringError(inconvertibleErrorCode(), "custom error"); +} + +TEST(VirtualOutputFileTest, construct) { + OutputFile F; + EXPECT_EQ("", F.getPath()); + EXPECT_FALSE(F); + EXPECT_FALSE(F.isOpen()); + +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) + EXPECT_DEATH(F.getOS(), "Expected open output stream"); +#endif +} + +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) +TEST(VirtualOutputFileTest, constructNull) { + EXPECT_DEATH(OutputFile("some/file/path", nullptr), + "Expected open output file"); +} +#endif + +TEST(VirtualOutputFileTest, destroy) { + MockOutputFileData Data; + StringRef FilePath = "some/file/path"; + + // Check behaviour when destroying, first without a handler and then with + // one. The handler shouldn't be called. + Optional F(std::in_place, FilePath, createMockOutput(Data)); + EXPECT_TRUE(F->isOpen()); + EXPECT_EQ(FilePath, F->getPath()); + EXPECT_EQ(Data.OS, &F->getOS()); +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(F.reset(), "output not closed"); +#endif + F->discardOnDestroy(Data.getHandler()); + EXPECT_EQ(0, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + F.reset(); + EXPECT_EQ(1, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + + // Try again when discard returns an error. This time the handler should be + // called. + Data.Discarder = createCustomError; + F.emplace("some/file/path", createMockOutput(Data)); + F->discardOnDestroy(Data.getHandler()); + F.reset(); + EXPECT_EQ(2, Data.Discarded); + EXPECT_EQ(1, Data.Handled); +} + +TEST(VirtualOutputFileTest, destroyProxy) { + MockOutputFileData Data; + + Optional F(std::in_place, "some/file/path", + createMockOutput(Data)); + F->discardOnDestroy(Data.getHandler()); + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F->createProxy().moveInto(Proxy), Succeeded()); + F.reset(); +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) + EXPECT_DEATH(*Proxy << "data", "use after reset"); +#endif + Proxy.reset(); +} + +TEST(VirtualOutputFileTest, discard) { + StringRef Content = "some data"; + MockOutputFileData Data; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.discard(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); + +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(consumeError(F.keep()), + "some/file/path: output already closed"); + EXPECT_DEATH(consumeError(F.discard()), + "some/file/path: output already closed"); +#endif + } + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); +} + +TEST(VirtualOutputFileTest, discardError) { + StringRef Content = "some data"; + MockOutputFileData Data; + Data.Discarder = createCustomError; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + EXPECT_THAT_ERROR(F.discard(), FailedWithMessage("custom error")); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + } + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); + EXPECT_EQ(0, Data.Handled); +} + +TEST(VirtualOutputFileTest, discardProxy) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.discard(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); +} + +TEST(VirtualOutputFileTest, discardProxyFlush) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F.getOS().SetBufferSize(Content.size() * 2); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ("", Data.V); + EXPECT_THAT_ERROR(F.discard(), Succeeded()); + EXPECT_EQ(Content, Data.V); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); +} + +TEST(VirtualOutputFileTest, keep) { + StringRef Content = "some data"; + MockOutputFileData Data; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.keep(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(consumeError(F.keep()), + "some/file/path: output already closed"); + EXPECT_DEATH(consumeError(F.discard()), + "some/file/path: output already closed"); +#endif + } + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); +} + +TEST(VirtualOutputFileTest, keepError) { + StringRef Content = "some data"; + MockOutputFileData Data; + Data.Keeper = createCustomError; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.keep(), FailedWithMessage("custom error")); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + } + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + EXPECT_EQ(0, Data.Handled); +} + +TEST(VirtualOutputFileTest, keepProxy) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + Proxy.reset(); + EXPECT_THAT_ERROR(F.keep(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); +} + +#if GTEST_HAS_DEATH_TEST +TEST(VirtualOutputFileTest, keepProxyStillOpen) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + EXPECT_DEATH(consumeError(F.keep()), "some/file/path: output has open proxy"); +} +#endif + +TEST(VirtualOutputFileTest, keepProxyFlush) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F.getOS().SetBufferSize(Content.size() * 2); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ("", Data.V); + Proxy.reset(); + EXPECT_THAT_ERROR(F.keep(), Succeeded()); + EXPECT_EQ(Content, Data.V); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); +} + +TEST(VirtualOutputFileTest, TwoProxies) { + StringRef Content = "some data"; + MockOutputFileData Data; + + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + // Can't have two open proxies at once. + { + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + EXPECT_THAT_ERROR( + F.createProxy().takeError(), + FailedWithMessage("some/file/path: output has open proxy")); + } + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + + // A second proxy after the first closes should work... + { + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + } +} + +} // end namespace