Index: llvm/include/llvm/Support/OutputManager.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/OutputManager.h @@ -0,0 +1,819 @@ +//===- llvm/Support/OutputManager.h - Output management ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_OUTPUTMANAGER_H +#define LLVM_SUPPORT_OUTPUTMANAGER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/simple_ilist.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace vfs { + +/// Configuration for output intent in \a OutputManager. +enum class ClientIntentOutputConfig { + NeedsSeeking, /// Require a stream that supports seeking. + NeedsReadAccess, /// Expect to read the file after writing. + + // Keep this last. + NumFlags, +}; + +/// Configuration for on-disk outputs in \a OutputManager. +enum class OnDiskOutputConfig { + UseTemporary, + UseTemporaryCreateMissingDirectories, + OpenFlagText, + RemoveFileOnSignal, + UseWriteThroughBuffer, + ForwardWriteThroughBuffer, + + // Keep this last. + NumFlags, +}; + +/// Combined configuration for outputs in \a OutputManager, combining the +/// different enum classes into a single enumeration. +class OutputConfigFlag { + /// Convert \c Flag to a raw unsigned value. + template constexpr static unsigned getRaw(FlagType Flag) { + return static_cast(Flag); + } + +public: + using UnderlyingType = unsigned char; + +private: + /// Convert \c Flag to \c UnderlyingType and offset it correctly. + constexpr static UnderlyingType getValue(ClientIntentOutputConfig Flag) { + // Put ClientIntentOutputConfig at the start. + return getRaw(Flag); + } + + /// Convert \c Flag to \c UnderlyingType and offset it correctly. + constexpr static UnderlyingType getValue(OnDiskOutputConfig Flag) { + // Put OnDiskOutputConfig after ClientIntentOutputConfig. + return getRaw(Flag) + getValue(ClientIntentOutputConfig::NumFlags); + } + +public: + /// Get the total number of flags in the combined enumeration. + constexpr static unsigned getNumFlags() { + return getRaw(ClientIntentOutputConfig::NumFlags) + + getRaw(OnDiskOutputConfig::NumFlags); + } + + /// Convert to the underlying type. + constexpr explicit operator UnderlyingType() const { return Flag; } + + /// Construct the combined enumeration from any individual enum. + template + constexpr OutputConfigFlag(FlagType Flag) : Flag(getValue(Flag)) {} + +private: + OutputConfigFlag() = delete; + + UnderlyingType Flag; +}; + +static_assert(std::numeric_limits::max() >= + OutputConfigFlag::getNumFlags(), + "Ran out of space for combined enums"); + +/// Shared code for \a PartialOutputConfig and \a OutputConfig. +class OutputConfigBase { +public: + /// Don't use std::bitset since it's mostly not constexpr. If primitive types + /// don't have enough bits, this can use a simple constexpr-friendly bitset. + using BitsetType = unsigned short; + + /// Construct the combined enumeration from any individual enum. + constexpr static BitsetType getBitset(OutputConfigFlag Flag) { + static_assert(sizeof(BitsetType) <= sizeof(unsigned), + "Returned literal will overflow"); + return 1u << static_cast(Flag); + } + + constexpr OutputConfigBase() = default; +}; + +static_assert(sizeof(OutputConfigBase::BitsetType) * 8 >= + OutputConfigFlag::getNumFlags(), + "Ran out of bits!"); + +/// Partial configuration for an output for use with \a OutputManager. Each +/// configuration flag can be \c true, \c false, or \a None. +class PartialOutputConfig : OutputConfigBase { + constexpr void setBits(BitsetType Bits, Optional Value) { + if (!Value) { + Mask &= ~Bits; + return; + } + + Mask |= Bits; + if (*Value) + Values |= Bits; + else + Values &= ~(Bits); + } + +public: + /// Check if anything is set. + constexpr bool empty() const { return !Mask; } + + /// Check the configuration value for \c Flag, if any. + constexpr Optional test(OutputConfigFlag Flag) const { + if (Mask & getBitset(Flag)) + return Values & getBitset(Flag); + return None; + } + + /// Set the configuration value for \c Flag to \c Value (dropping the + /// configuration if \c Value is \c None). + constexpr PartialOutputConfig &set(OutputConfigFlag Flag, + Optional Value = true) { + setBits(getBitset(Flag), Value); + return *this; + } + + /// Set the configuration values for \c FlagsToChange to \c Value (dropping + /// the configuration if \c Value is \c None). + constexpr PartialOutputConfig & + set(std::initializer_list FlagsToChange, + Optional Value = true) { + BitsetType ChangeMask = 0; + for (OutputConfigFlag Flag : FlagsToChange) + ChangeMask |= getBitset(Flag); + + setBits(ChangeMask, Value); + return *this; + } + + /// Set the configuration value for \c Flag to \c false. + constexpr PartialOutputConfig &reset(OutputConfigFlag Flag) { + return set(Flag, false); + } + + /// Set the configuration values for \c FlagsToChange to \c false. + constexpr PartialOutputConfig & + reset(std::initializer_list FlagsToChange) { + return set(FlagsToChange, false); + } + + /// Drop the configuration for \c Flag (set it to None). + constexpr PartialOutputConfig &drop(OutputConfigFlag Flag) { + return set(Flag, None); + } + + /// Drop the configuration for \c FlagsToChange (set them to None). + constexpr PartialOutputConfig & + drop(std::initializer_list FlagsToChange) { + return set(FlagsToChange, None); + } + + /// Merge this partial configuration with \c Overrides, where the value in \c + /// Overrides wins if one is \c true and the other \c false. + constexpr PartialOutputConfig &applyOverrides(PartialOutputConfig Overrides) { + Mask |= Overrides.Mask; + Values |= Overrides.Mask & Overrides.Values; + Values &= ~Overrides.Mask | Overrides.Values; + return *this; + } + + /// Merge this partial configuration with \c Defaults, where the existing + /// value wins if one is \c true and the other \c false. + constexpr PartialOutputConfig &applyDefaults(PartialOutputConfig Defaults) { + return *this = Defaults.applyOverrides(*this); + } + + /// Nothing is set. + constexpr PartialOutputConfig() = default; + +private: + friend class OutputConfig; + BitsetType Values = 0; + BitsetType Mask = 0; +}; + +/// Full configuration for an output for use by the \a OutputManager. Each +/// configuration flag is either \c true or \c false. +class OutputConfig : OutputConfigBase { +public: + /// Test whether there are no flags turned on. + constexpr bool none() const { return !Flags; } + + /// Check the value for \c Flag. + constexpr bool test(OutputConfigFlag Flag) const { + return Flags & getBitset(Flag); + } + + /// Set \c Flag to \c Value. + constexpr OutputConfig &set(OutputConfigFlag Flag, bool Value = true) { + if (Value) + Flags |= getBitset(Flag); + else + Flags &= ~getBitset(Flag); + return *this; + } + + /// Set \c FlagsToChange to \c Value. + constexpr OutputConfig & + set(std::initializer_list FlagsToChange, + bool Value = true) { + return applyOverrides(PartialOutputConfig().set(FlagsToChange, Value)); + } + + /// Set \c Flag to \c false. + constexpr OutputConfig &reset(OutputConfigFlag Flag) { + return set(Flag, false); + } + + /// Set \c FlagsToChange to \c false. + constexpr OutputConfig & + reset(std::initializer_list FlagsToChange) { + return set(FlagsToChange, false); + } + + /// Apply overrides from the partial configuration \p Overrides. Takes the + /// value of any flag set in \c Overrides, leaving alone any flag where \p + /// Overrides has \c None. + constexpr OutputConfig &applyOverrides(PartialOutputConfig Overrides) { + Flags |= Overrides.Mask & Overrides.Values; + Flags &= ~Overrides.Mask | Overrides.Values; + return *this; + } + + /// Nothing is set. + constexpr OutputConfig() = default; + + /// Set exactly the flags listed, leaving others turned off. + constexpr OutputConfig(std::initializer_list OnFlags) { + set(OnFlags); + } + +private: + BitsetType Flags = 0; +}; + +class OutputDestination; +class OutputBackend; + +/// Opaque description of a compiler output that has been created. +class Output { +public: + /// Close an output, finalizing its content and sending it to the configured + /// \a OutputBackend. If \p ShouldErase is set to \c true, redirects to \a + /// erase() instead. + /// + /// This destructs the stream if it still owns it and writes the output to + /// its final destination(s). E.g., for \a OnDiskOutputBackend, any temporary + /// files will get moved into place, or for \a InMemoryOutputBackend, the \a + /// MemoryBuffer will be created and stored in the \a InMemoryFileSystem. + /// + /// \pre If \a takeOS() has been called, the stream should have been + /// destructed before calling this function. + /// \pre \a isOpen(); i.e., neither \a erase() nor \a close() has been + /// called yet. + Error close(bool ShouldErase = false); + + /// Erase the output. + /// + /// \pre \a isOpen(); i.e., neither \a erase() nor \a close() has been + /// called yet. + void erase(); + + /// Check if \a erase() or \a close() has already been called. + bool isOpen() const { return IsOpen; } + + /// Get a pointer to the output stream, if it hasn't been taken. + raw_pwrite_stream *getOS() const { return OS.get(); } + + /// Take the output stream. This stream should be destructed before calling + /// \a close(). + /// + /// \post \a getOS() returns \c nullptr. + std::unique_ptr takeOS() { return std::move(OS); } + + /// Get the configuration for this output. + OutputConfig getConfig() const { return Config; } + + /// Get the output path for this output. + StringRef getPath() const { return Path; } + + /// Erases the output if it hasn't already been closed. + ~Output(); + + /// Constructor for an output. \p Dest must be a newly initialized \a + /// OutputDestination, or \c nullptr. + /// + /// \post \a getOS() returns a pointer to the stream for writing to \p Dest + /// (\a raw_null_ostream if \p Dest is \c nullptr). This may write to \a + /// Bytes as an intermediate buffer. + Output(StringRef Path, OutputConfig Config, + std::unique_ptr Dest) + : Path(Path.str()), Config(Config), Dest(std::move(Dest)) { + initializeOS(); + } + +private: + /// Helper for \a Output::Output(). + void initializeOS(); + + std::string Path; + OutputConfig Config; + + /// Tracks whether the output is still open, before one of \a erase() or \a + /// close() is called. + bool IsOpen = true; + + /// Content buffer if the output destination requests one. + Optional> Bytes; + + /// The target for this output, provided by \a OutputManager's \a + /// OutputBackend. + std::unique_ptr Dest; + + // Destroy before Dest and ContentBuffer since it could reference them in its + // destructor. + std::unique_ptr OS; +}; + +/// Manager for outputs, handling safe and atomic creation of outputs. \a +/// OutputManager accepts an arbitrary \a OutputBackend, which may write files +/// on disk, store buffers in an \a InMemoryFileSystem, send the bytes to other +/// arbitrary destinations, or some combination of the above. +class OutputManager { +public: + /// Get the default congiuration for outputs. + const OutputConfig &getDefaults() const { return Defaults; } + OutputConfig &getDefaults() { return Defaults; } + + /// Open a new output for writing and add it to the list of tracked + /// outputs. + /// + /// \param OutputPath - If given, the path to the output. "-" indicates + /// stdout. + /// \return An error, or a valid \a Output that is open and ready for + /// content. + Expected> + createOutput(StringRef OutputPath, PartialOutputConfig Overrides = {}); + + /// Change the current OutputBackend, to be used for \a createOutput() until + /// it's next changed. + void setBackend(IntrusiveRefCntPtr Backend) { + this->Backend = std::move(Backend); + } + + /// Check whether there is current an \a OutputBackend installed. + bool hasBackend() const { return bool(Backend); } + + /// Get a reference to the current backend. + /// + /// \pre \a hasBackend() is \c true. + OutputBackend &getBackend() const { + assert(hasBackend() && "OutputManager missing a backend?"); + return *Backend; + } + + /// Take the current backend. + /// + /// \post \a hasBackend() is \c false. + IntrusiveRefCntPtr takeBackend() { return std::move(Backend); } + + /// Initialize with a newly constructed \a OnDiskOutputBackend. + OutputManager(); + + /// Initialize with a custom backend. + explicit OutputManager(IntrusiveRefCntPtr Backend) + : Backend(std::move(Backend)) {} + +private: + /// Default configuration for files that aren't specifically configured. Can + /// be modified with \a getDefaults(). + OutputConfig Defaults = { + ClientIntentOutputConfig::NeedsSeeking, + ClientIntentOutputConfig::NeedsReadAccess, + OnDiskOutputConfig::UseTemporary, + OnDiskOutputConfig::RemoveFileOnSignal, + }; + + IntrusiveRefCntPtr Backend; +}; + +/// RAII utility for temporarily swapping an \a OutputManager's backend. +class ScopedOutputManagerBackend { +public: + /// Install \p TemporaryBackend as \p M's backend. + ScopedOutputManagerBackend(OutputManager &M, + IntrusiveRefCntPtr TemporaryBackend) + : M(M), OriginalBackend(M.takeBackend()) { + M.setBackend(std::move(TemporaryBackend)); + } + + /// Restore the original backend to the \a OutputManager. + ~ScopedOutputManagerBackend() { M.setBackend(std::move(OriginalBackend)); } + +private: + OutputManager &M; + IntrusiveRefCntPtr OriginalBackend; +}; + +/// Backend interface for \a OutputManager. Its job is to generate \a +/// OutputDestination given an \a OutputPath and \c OutputConfig. +class OutputBackend : public RefCountedBase { + virtual void anchor(); + +public: + /// Create an output destination, suitable for initializing \a Output. + /// Returning \c nullptr indicates that this backend is ignoring the output. + /// + /// \return A valid \a OutputDestination, \c nullptr, or an \a Error. + virtual Expected> + createDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) = 0; + + virtual ~OutputBackend() = default; +}; + +/// Interface for managing the destination of an \a Output. Most users only +/// need to deal with \a Output. +/// +/// \a OutputDestination's lifetime is expected to follow one of the following +/// "good" paths after construction: +/// - \a takeStream() is called and returns a valid stream. The caller writes +/// content to the stream, destructs it, and then calls \a +/// storeStreamedContent() is to store the content. +/// - \a takeStream() is called and returns \c nullptr. The caller collects the +/// content and calls \a storeContent() to store it. +/// - \a storeContent() is called without first calling \a takeStream(), as the +/// caller wants to pass in the completed content as a whole instead of +/// streaming. +/// +/// If the destination is destructed before calling \a storeStreamedContent() +/// and \a storeContent(), this output will be cancelled and temporaries +/// cleaned up. +/// +/// \a storeContent() is designed to allow output destinations to be chained, +/// passing content between them. \a ContentBuffer helps to manage the lifetime +/// of the content, copying data and constructing memory buffers only as +/// needed. +class OutputDestination { + virtual void anchor(); + +public: + /// Utility for holding completed content for an output. + struct ContentBuffer { + StringRef getPath() const { + assert(Buffer && "Expected initialization?"); + return Buffer->getBufferIdentifier(); + } + StringRef getBytes() const { + assert(Buffer && "Expected initialization?"); + return Vector ? StringRef(Vector->begin(), Vector->size()) + : Buffer->getBuffer(); + } + + /// Whether the content is currently owned by this utility. + bool ownsContent() const { return Vector || OwnedBuffer; } + + /// Returns a valid \a MemoryBuffer. If \a ownsContent() is false, + /// this buffer is a reference constructed on the fly using \a + /// MemoryBuffer::getMemBuffer(). + /// + /// \post \b ownsContent() is false. + std::unique_ptr takeBuffer(); + + /// Returns a valid \a MemoryBuffer that owns its bytes, or \c + /// nullptr if \a ownsContent() is \c false. + /// + /// \post \b ownsContent() is false. + std::unique_ptr takeOwnedBufferOrNull(); + + /// Returns a valid \a MemoryBuffer that owns its bytes, or calls + /// MemoryBuffer::getMemBufferCopy() if \a ownsContent() is \c false. + /// + /// \post \b ownsContent() is false. + std::unique_ptr takeOwnedBufferOrCopy(); + + /// Construct a reference to content owned by someone else. + /// + /// \post \b ownsContent() is false. + ContentBuffer(MemoryBufferRef Buffer) : Buffer(Buffer) { + assert(!this->Buffer->getBufferEnd()[0] && "Requires null terminator"); + } + + /// Construct a buffer named \p Identifier from \p Vector. This buffer is + /// created lazily on the first call to \a takeBuffer() and friends using + /// \a SmallVectorMemoryBuffer. + /// + /// \post \b ownsContent() is true, unless \c Vector.empty(). + /// \post \a getBytes() returns a range with the memory from \p Vector, + /// unless \p Vector was empty or in small mode. + ContentBuffer(SmallVectorImpl &&Vector, StringRef Identifier) + : Vector(std::move(Vector)) { + finishConstructingFromVector(Identifier); + assert(!Buffer->getBufferEnd()[0] && "Requires null terminator"); + } + + /// Store \p Buffer as the content, returned by the first call to \a + /// takeBuffer() and friends. + /// + /// \pre \p Buffer is null-terminated. + /// \post \b ownsContent() is true. + ContentBuffer(std::unique_ptr Buffer) + : Buffer(*Buffer), OwnedBuffer(std::move(Buffer)) { + assert(!this->Buffer->getBufferEnd()[0] && "Requires null terminator"); + } + + ContentBuffer(ContentBuffer &&) = default; + ContentBuffer &operator=(ContentBuffer &&) = default; + + private: + ContentBuffer() = delete; + ContentBuffer(const ContentBuffer &&) = delete; + ContentBuffer &operator=(const ContentBuffer &&) = delete; + + /// Main body of constructor when initializing \a Vector. + void finishConstructingFromVector(StringRef Identifier); + + /// Owned content stored in a vector. + Optional> Vector; + + /// Always initialized; only optional to facilite the small vector + /// constructor. + Optional Buffer; + + /// Owned content stored in a memory buffer. + std::unique_ptr OwnedBuffer; + }; + + /// Take the output stream directly. Can return \c nullptr if this + /// destination requires a content buffer. + /// + /// If this function is called and \c nullptr is NOT returned, then once the + /// stream has been filled with content \a storeStreamedContent() should be + /// called to finalize it. + /// + /// If \c nullptr is returned, \a storeContent() should be called instead of + /// \a storeStreamedContent() with a filled content buffer. + std::unique_ptr takeStream() { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + assert(IsOpen); + assert(!TookStream); + TookStream = true; +#endif + std::unique_ptr Stream; + if (!Next) + Stream = takeStreamImpl(); +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + RequestedContentBuffer = !Stream; +#endif + return Stream; + } + + /// Store streamed content. Only valid to call if \a takeStream() was called + /// and it did not return \c nullptr. + Error storeStreamedContent() { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + assert(IsOpen); + assert(TookStream); + assert(!Next); + assert(!RequestedContentBuffer); + IsOpen = false; +#endif + return storeStreamedContentImpl(); + } + + /// Store \a Content in the output destination, and then forward it the next + /// one in the chain, if any. Not valid to call if \a takeStream() was called + /// and returned a stream. + /// + /// Calls \a storeContentImpl() with a reference to \p Content, then forwards + /// it to \a Next. + Error storeContent(ContentBuffer Content) { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + assert(IsOpen); + assert(!TookStream || RequestedContentBuffer); + IsOpen = false; +#endif + if (Error E = storeContentImpl(Content)) + return E; + if (Next) + return Next->storeContent(std::move(Content)); + return Error::success(); + } + +protected: + /// Override this to allow content to be written directly to a stream, rather + /// than collected in a content buffer. + virtual std::unique_ptr takeStreamImpl() { + return nullptr; + } + + /// If \a takeStreamImpl() is called and does not return \c nullptr, this + /// will be called when content should be flushed. + virtual Error storeStreamedContentImpl() { + llvm_unreachable("override this when overriding takeStreamImpl"); + } + + /// All destinations must know how to handle a content buffer. + virtual Error storeContentImpl(ContentBuffer &Content) = 0; + + /// All destinations must support being constructed in front of a chain + /// of destinations. + explicit OutputDestination(std::unique_ptr Next) + : Next(std::move(Next)) {} + +public: + virtual ~OutputDestination() = default; + +private: +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + bool IsOpen = true; + bool TookStream = false; + bool RequestedContentBuffer = false; +#endif + + std::unique_ptr Next; +}; + +/// Base class for OutputManager errors. +class OutputError : public ErrorInfo { + void anchor() override; + +public: + StringRef getOutputPath() const { return OutputPath; } + std::error_code getErrorCode() const { return EC; } + + std::error_code convertToErrorCode() const override { return getErrorCode(); } + + // Used by ErrorInfo::classID. + static char ID; + +protected: + OutputError(StringRef OutputPath, std::error_code EC) + : OutputPath(OutputPath), EC(EC) { + assert(EC && "Cannot create OutputError from success EC"); + } + +private: + std::string OutputPath; + std::error_code EC; +}; + +/// The output already exists in filesystem and it cannot be overwritten. +class CannotOverwriteExistingOutputError final + : public ErrorInfo { + friend Error createCannotOverwriteExistingOutputError(StringRef OutputPath); + + void anchor() override; + +public: + void log(raw_ostream &OS) const override { + OS << "'" << getOutputPath() << "' already exists in-memory"; + } + + // Used by ErrorInfo::classID. + static char ID; + +private: + CannotOverwriteExistingOutputError(StringRef OutputPath) + : ErrorInfo( + OutputPath, + std::make_error_code(std::errc::device_or_resource_busy)) {} +}; + +inline Error createCannotOverwriteExistingOutputError(StringRef OutputPath) { + return Error(std::unique_ptr( + new CannotOverwriteExistingOutputError(OutputPath))); +} + +/// A temporary file could not be renamed to the final output on disk. +class OnDiskOutputRenameTempError final + : public ErrorInfo { + friend Error createOnDiskOutputRenameTempError(StringRef TempPath, + StringRef OutputPath, + std::error_code EC); + + void anchor() override; + +public: + void log(raw_ostream &OS) const override { + assert(getErrorCode()); + OS << "'" << TempPath << "' could not be renamed to "; + OS << "'" << getOutputPath() << "': "; + OS << getErrorCode().message(); + } + + StringRef getTempPath() const { return TempPath; } + + // Used by ErrorInfo::classID. + static char ID; + +private: + OnDiskOutputRenameTempError(StringRef TempPath, StringRef OutputPath, + std::error_code EC) + : ErrorInfo(OutputPath, EC), + TempPath(TempPath) { + assert(EC && "Cannot create OnDiskOutputRenameTempError from success EC"); + } + + std::string TempPath; +}; + +inline Error createOnDiskOutputRenameTempError(StringRef TempPath, + StringRef OutputPath, + std::error_code EC) { + return Error(std::unique_ptr( + new OnDiskOutputRenameTempError(TempPath, OutputPath, EC))); +} + +class OnDiskOutputBackend : public OutputBackend { + void anchor() override; + +public: + Expected> + createDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) override; + + /// Big enough that mmap won't use up too much address space. + static constexpr unsigned MinimumSizeToReturnWriteThroughBuffer = 4 * 4096; +}; + +/// Create a backend that ignores all output. +IntrusiveRefCntPtr makeNullOutputBackend(); + +/// Create a backend that mirrors content between \a Backend1 and \a +/// Backend2. +/// +/// Both backends are asked to create destinations for each output. \a Backend1 +/// receives the content before \a Backend2. +IntrusiveRefCntPtr +makeMirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2); + +/// A backend for storing outputs in an instance of \a InMemoryFileSystem. +class InMemoryOutputBackend : public OutputBackend { + void anchor() override; + +public: + Expected> + createDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) override; + + /// Install an in-memory filesystem for writing outputs. + /// + /// Modify \a getDefaults() to turn on this use of \p FS by default, + /// otherwise it will only be used for files with custom configuration. + void setInMemoryFS(IntrusiveRefCntPtr FS) { + assert(FS && "expected valid file-system"); + InMemoryFS = std::move(FS); + } + + /// Get the installed in-memory filesystem, if any. + InMemoryFileSystem &getInMemoryFS() const { return *InMemoryFS; } + + bool shouldStoredBufferBeOwned() const { return ShouldStoredBufferBeOwned; } + void setShouldStoredBufferBeOwned(bool ShouldOwn) { + ShouldStoredBufferBeOwned = ShouldOwn; + } + + bool shouldRejectExistingFiles() const { return ShouldRejectExistingFiles; } + void setShouldRejectExistingFiles(bool RejectExisting) { + ShouldRejectExistingFiles = RejectExisting; + } + + InMemoryOutputBackend(IntrusiveRefCntPtr FS) { + setInMemoryFS(std::move(FS)); + } + +private: + bool ShouldRejectExistingFiles = false; + bool ShouldStoredBufferBeOwned = false; + + /// In-memory filesystem for writing outputs to. + IntrusiveRefCntPtr InMemoryFS; +}; + +/// Create an adaptor backend that filters the outputs that written to the +/// underlying backend. Outputs where \p Filter returns \c false will be +/// ignored. +IntrusiveRefCntPtr makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + unique_function Filter); + +} // namespace vfs +} // namespace llvm + +#endif // LLVM_SUPPORT_OUTPUTMANAGER_H Index: llvm/lib/Support/CMakeLists.txt =================================================================== --- llvm/lib/Support/CMakeLists.txt +++ llvm/lib/Support/CMakeLists.txt @@ -148,6 +148,7 @@ NativeFormatting.cpp OptimizedStructLayout.cpp Optional.cpp + OutputManager.cpp Parallel.cpp PluginLoader.cpp PrettyStackTrace.cpp Index: llvm/lib/Support/OutputManager.cpp =================================================================== --- /dev/null +++ llvm/lib/Support/OutputManager.cpp @@ -0,0 +1,691 @@ +//===- OutputManager.cpp - Manage compiler outputs ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the OutputManager interface. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/OutputManager.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::vfs; + +void OutputError::anchor() {} +char OutputError::ID = 0; + +void CannotOverwriteExistingOutputError::anchor() {} +char CannotOverwriteExistingOutputError::ID = 0; + +void OnDiskOutputRenameTempError::anchor() {} +char OnDiskOutputRenameTempError::ID = 0; + +Output::~Output() = default; + +void Output::initializeOS() { + if (!Dest) { + OS = std::make_unique(); + return; + } + OS = Dest->takeStream(); + if (!OS) { + // This destination needs a content buffer. + Bytes.emplace(); + OS = std::make_unique(*Bytes); + } +} + +Error Output::close(bool ShouldErase) { + assert(isOpen() && "Output already closed or erased"); + + // Forward to Output::erase() if necessary. + if (ShouldErase) { + erase(); + return Error::success(); + } + + // Mark as closed and destruct the stream if we still own it. + IsOpen = false; + OS = nullptr; + + if (!Dest) + return Error::success(); + + // Clear the destination on exit. + std::unique_ptr MovedDest = std::move(Dest); + + if (!Bytes) + return MovedDest->storeStreamedContent(); + return MovedDest->storeContent( + OutputDestination::ContentBuffer(std::move(*Bytes), Path)); +} + +void Output::erase() { + // Release resources in destruction order to erase. + assert(isOpen() && "Output already closed or erased"); + IsOpen = false; + + // Close the stream first in case its destructor references Bytes or Dest. + OS = nullptr; + + // Erase any temporary content from the destination. + Dest = nullptr; + + // Free the content buffer. + Bytes = None; +} + +void OutputDestination::anchor() {} +void OutputBackend::anchor() {} + +void OutputDestination::ContentBuffer::finishConstructingFromVector( + StringRef Identifier) { + // If Vector is empty, we can't guarantee SmallVectorMemoryBuffer will keep + // pointer identity. But since there's no content, it's all moot; just + // construct a reference. + if (Vector->empty()) { + Vector = None; + Buffer.emplace("", Identifier); + return; + } + + // Ensure null-termination. + Vector->push_back(0); + Vector->pop_back(); + Buffer.emplace(StringRef(Vector->begin(), Vector->size()), Identifier); +} + +std::unique_ptr +OutputDestination::ContentBuffer::takeOwnedBufferOrNull() { + assert(Buffer && "Expected initialization?"); + if (OwnedBuffer) + return std::move(OwnedBuffer); + + if (!Vector) + return nullptr; + + bool IsEmpty = Vector->empty(); + (void)IsEmpty; + auto VectorBuffer = std::make_unique( + std::move(*Vector), Buffer->getBufferIdentifier()); + Vector = None; + assert(Buffer->getBufferStart() == VectorBuffer->getBufferStart()); + assert(Buffer->getBufferEnd() == VectorBuffer->getBufferEnd()); + return VectorBuffer; +} + +std::unique_ptr OutputDestination::ContentBuffer::takeBuffer() { + assert(Buffer && "Expected initialization?"); + if (std::unique_ptr B = takeOwnedBufferOrNull()) + return B; + return MemoryBuffer::getMemBuffer(*Buffer); +} + +std::unique_ptr +OutputDestination::ContentBuffer::takeOwnedBufferOrCopy() { + assert(Buffer && "Expected initialization?"); + if (std::unique_ptr B = takeOwnedBufferOrNull()) + return B; + return MemoryBuffer::getMemBufferCopy(Buffer->getBuffer(), + Buffer->getBufferIdentifier()); +} + +OutputManager::OutputManager() + : OutputManager(makeIntrusiveRefCnt()) {} + +Expected> +OutputManager::createOutput(StringRef OutputPath, + PartialOutputConfig Overrides) { + OutputConfig Config = Defaults; + Config.applyOverrides(Overrides); + + Expected> ExpectedDest = + getBackend().createDestination(OutputPath, Config, nullptr); + if (!ExpectedDest) + return ExpectedDest.takeError(); + + return std::make_unique(OutputPath, Config, std::move(*ExpectedDest)); +} + +namespace { +class NullOutputBackend : public OutputBackend { +public: + Expected> + createDestination(StringRef, OutputConfig, + std::unique_ptr NextDest) final { + return std::move(NextDest); + } +}; +} // anonymous namespace + +IntrusiveRefCntPtr llvm::vfs::makeNullOutputBackend() { + return makeIntrusiveRefCnt(); +} + +namespace { +class OnDiskOutputDestination : public OutputDestination { +public: + /// Open a file on-disk for writing to \a OutputPath. + /// + /// This calls \a initializeFD() to initialize \a FD (see that + /// documentation for more detail) and \a UseWriteThroughBuffer. Unless \a + /// UseWriteThroughBuffer was determined to be \c true, this function will + /// then initialize \a OS with a valid stream. + Error initializeFile(); + + /// Erases the file if it hasn't be closed. If \a TempFile is set, erases it; + /// otherwise, erases \a OutputPath. + ~OnDiskOutputDestination() override; + + /// Take an open output stream. + std::unique_ptr takeStreamImpl() override; + + Error storeStreamedContentImpl() final { + return closeFile(OutputPath, nullptr); + } + + Error storeContentImpl(ContentBuffer &Content) final { + return closeFile(Content.getPath(), &Content); + } + +private: + /// Returns true if the final write uses a \a + /// WriteThroughMemoryBuffer. This is only valid to call after \a + /// initializeFile(). + bool useWriteThroughBuffer() const { + assert(UseWriteThroughBuffer != None); + return *UseWriteThroughBuffer; + } + + using CheckingContentConsumerType = + function_ref)>; + + /// Close a file after successfully collecting the output. + /// + /// The output content must have already been written somewhere, but + /// exactly where depends on the configuration. + /// + /// - If \a WriteThroughMemoryBuffer is \c true, the output will be in \a + /// Content and \a FD has an open file descriptor. In that case, the file + /// will be completed by opening \a WriteThroughMemoryBuffer and + /// calling \a std::memcpy(). + /// - Else if the output is in \a Content, it will be written to \a OS, + /// which should be an already-open file stream, and the content will be + /// written there and the stream destroyed to flush it. + /// - Else the content should already be in the file. + /// + /// Once the content is on-disk, if \a TempPath is set, this calls \a + /// sys::rename() to move the file to \a OutputPath. + /// + /// Returns a file-backed MemoryBuffer when the file was written using a + /// write-through memory buffer, unless it won't be null-terminated or it's + /// too small. Otherwise, when there's no error, returns \c nullptr. + Error closeFile(StringRef OutputPath, ContentBuffer *MaybeContent); + + Expected> + openMappedFile(size_t Size); + + std::unique_ptr convertMappedFileToBuffer( + std::unique_ptr Mapping); + + /// Attempt to open a temporary file for \a OutputPath. + /// + /// This tries to open a uniquely-named temporary file for \a OutputPath, + /// possibly also creating any missing directories if \a + /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a + /// Config. + /// + /// \post FD and \a TempPath are initialized if this is successful. + std::error_code tryToCreateTemporary(); + + /// Open a file on-disk for writing to OutputPath. + /// + /// This opens a file for writing and assigns the file descriptor to \c FD. + /// Exactly how the file is opened depends on the \a OnDiskOutputConfig + /// settings in \p Config and (if the file exists) the type of file. + /// + /// - If \p outputPath is \c "-" (indicating stdin), this function returns + /// \a Error::success() but has no effect. + /// - \a OnDiskOutputConfig::UseTemporary requests that a temporary file is + /// opened first, to be renamed to \p OutputPath when \a closeFile() is + /// called. This is disabled if \p OutputPath exists and \a + /// sys::fs::is_regular_file() returns \c false (such as a named + /// pipe). If \a tryToCreateTemporary() fails, this falls back to + /// no-temp-file mode. (Even if using a temporary file, \p OutputPath is + /// checked for write permission.) + /// - \a OnDiskOutputConfig::RemoveFileOnSignal installs a signal handler + /// to remove the opened file. + /// + /// This function also initializes \a UseWriteThroughBuffer, depending on \a + /// OnDiskOutputConfig::UseWriteThroughBuffer and whether the file type + /// supports them. + /// + /// \post UseWriteThroughBuffer is set. + /// \post FD is set unless \a OutputPath is \c "-" or on error. + /// \post TempPath is set if a temporary file was opened successfully. + Error initializeFD(); + +public: + OnDiskOutputDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr Next) + : OutputDestination(std::move(Next)), OutputPath(OutputPath.str()), + Config(Config) {} + +private: + std::string OutputPath; + OutputConfig Config; + Optional UseWriteThroughBuffer; + bool IsOpen = true; + std::unique_ptr OS; + Optional TempPath; + Optional FD; +}; +} // anonymous namespace + +void OnDiskOutputBackend::anchor() {} + +std::error_code OnDiskOutputDestination::tryToCreateTemporary() { + // Create a temporary file. + // Insert -%%%%%%%% before the extension (if any), and because some tools + // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build + // artifacts, also append .tmp. + StringRef OutputExtension = sys::path::extension(OutputPath); + SmallString<128> TempPath = + StringRef(OutputPath).drop_back(OutputExtension.size()); + TempPath += "-%%%%%%%%"; + TempPath += OutputExtension; + TempPath += ".tmp"; + + auto tryToCreateImpl = [&]() { + int NewFD; + if (std::error_code EC = + sys::fs::createUniqueFile(TempPath, NewFD, TempPath)) + return EC; + if (Config.test(OnDiskOutputConfig::RemoveFileOnSignal)) + sys::RemoveFileOnSignal(TempPath); + this->TempPath = TempPath.str().str(); + FD = NewFD; + return std::error_code(); + }; + + // Try to create the temporary. + std::error_code EC = tryToCreateImpl(); + if (!EC) + return std::error_code(); + + if (EC != std::errc::no_such_file_or_directory || + !Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)) + return EC; + + // Create parent directories and try again. + StringRef ParentPath = sys::path::parent_path(OutputPath); + if ((EC = sys::fs::create_directories(ParentPath))) + return EC; + return tryToCreateImpl(); +} + +Error OnDiskOutputDestination::initializeFD() { + // Disable temporary file for stdout (and return early since we won't use a + // file descriptor directly). + if (OutputPath == "-") { + UseWriteThroughBuffer = false; + return Error::success(); + } + + // Initialize UseWriteThroughBuffer. + UseWriteThroughBuffer = + Config.test(OnDiskOutputConfig::UseWriteThroughBuffer); + + // Function to check and update whether to use a write-through buffer. + bool CheckedStatusForWriteThrough = false; + auto checkStatusForWriteThrough = [&](const sys::fs::file_status &Status) { + if (CheckedStatusForWriteThrough) + return; + sys::fs::file_type Type = Status.type(); + if (Type != sys::fs::file_type::regular_file && + Type != sys::fs::file_type::block_file) + UseWriteThroughBuffer = false; + CheckedStatusForWriteThrough = true; + }; + + // Disable temporary file for other non-regular files, and if we get a status + // object, also check if we can write and disable write-through buffers if + // appropriate. + if (Config.test(OnDiskOutputConfig::UseTemporary)) { + sys::fs::file_status Status; + sys::fs::status(OutputPath, Status); + if (sys::fs::exists(Status)) { + if (!sys::fs::is_regular_file(Status)) + Config.reset(OnDiskOutputConfig::UseTemporary); + + // Fail now if we can't write to the final destination. + if (!sys::fs::can_write(OutputPath)) + return errorCodeToError( + std::make_error_code(std::errc::operation_not_permitted)); + + checkStatusForWriteThrough(Status); + } + } + + // If (still) using a temporary file, try to create it (and return success if + // that works). + if (Config.test(OnDiskOutputConfig::UseTemporary)) + if (!tryToCreateTemporary()) + return Error::success(); + + // Not using a temporary file. Open the final output file. + int NewFD; + if (auto EC = sys::fs::openFileForWrite( + OutputPath, NewFD, sys::fs::CD_CreateAlways, + Config.test(OnDiskOutputConfig::OpenFlagText) ? sys::fs::OF_Text + : sys::fs::OF_None)) + return errorCodeToError(EC); + FD = NewFD; + + // Check the status with the open FD to see whether a write-through buffer + // makes sense. + if (*UseWriteThroughBuffer && !CheckedStatusForWriteThrough) { + sys::fs::file_status Status; + sys::fs::status(NewFD, Status); + checkStatusForWriteThrough(Status); + } + + if (Config.test(OnDiskOutputConfig::RemoveFileOnSignal)) + sys::RemoveFileOnSignal(OutputPath); + return Error::success(); +} + +Error OnDiskOutputDestination::initializeFile() { + if (Error E = initializeFD()) + return E; + + assert(FD || OutputPath == "-"); + if (useWriteThroughBuffer()) + return Error::success(); + + // Open the raw_fd_ostream right away to free the file descriptor. + std::error_code EC; + OS = OutputPath == "-" + ? std::make_unique(OutputPath, EC) + : std::make_unique(*FD, /*shouldClose=*/true); + assert(!EC && "Unexpected error opening stdin"); + + return Error::success(); +} + +std::unique_ptr OnDiskOutputDestination::takeStreamImpl() { + if (useWriteThroughBuffer()) + return nullptr; + + assert(OS && "Expected file to be initialized"); + + // Check whether we can get away with returning the stream directly. + if (OS->supportsSeeking() || + !Config.test(ClientIntentOutputConfig::NeedsSeeking)) + return std::move(OS); + + // Wrap the raw_fd_ostream with a buffer if necessary. + return std::make_unique(std::move(OS)); +} + +std::unique_ptr +OnDiskOutputDestination::convertMappedFileToBuffer( + std::unique_ptr Mapping) { + assert(Mapping->const_data()[Mapping->size()] == 0 && + "Expected mmap region to be null-terminated"); + + class MappedMemoryBuffer : public llvm::MemoryBuffer { + public: + StringRef getBufferIdentifier() const override { return OutputPath; } + BufferKind getBufferKind() const override { return MemoryBuffer_MMap; } + + MappedMemoryBuffer(const std::string &OutputPath, + std::unique_ptr Mapping) + : OutputPath(OutputPath), Mapping(std::move(Mapping)) { + const char *Data = this->Mapping->const_data(); + init(Data, Data + this->Mapping->size(), /*RequiresNullTerminator=*/true); + } + + private: + std::string OutputPath; + std::unique_ptr Mapping; + }; + return std::make_unique(OutputPath, std::move(Mapping)); +} + +Expected> +OnDiskOutputDestination::openMappedFile(size_t Size) { + if (auto EC = sys::fs::resize_file_before_mapping_readwrite(*FD, Size)) + return errorCodeToError(EC); + + std::error_code EC; + auto Mapping = std::make_unique( + sys::fs::convertFDToNativeFile(*FD), + sys::fs::mapped_file_region::readwrite, Size, 0, EC); + if (EC) + return errorCodeToError(EC); + return Mapping; +} + +Error OnDiskOutputDestination::closeFile(StringRef OutputPath, + ContentBuffer *MaybeContent) { + assert(IsOpen && "can't close a file twice"); + IsOpen = false; + assert((!MaybeContent || MaybeContent->getPath() == OutputPath) && + "Expected buffer identifier to match output path"); + + Optional BufferedContent; + if (MaybeContent) + BufferedContent = MaybeContent->getBytes(); + if (useWriteThroughBuffer()) { + assert(FD && "Write-through buffer needs a file descriptor"); + assert(!OS && "Expected no stream when using write-through buffer"); + assert(BufferedContent && "Expected buffered content"); + std::unique_ptr Mapping; + if (auto ExpectedMapping = openMappedFile(BufferedContent->size())) + Mapping = std::move(*ExpectedMapping); + else + return ExpectedMapping.takeError(); + + std::memcpy(Mapping->data(), BufferedContent->begin(), + BufferedContent->size()); + + // Decide whether to return the file-backed buffer. Skip it when it's too + // small (fragmenting memory) or page-aligned (missing a null-terminator). + static unsigned PageSize = sys::Process::getPageSizeEstimate(); + if (MaybeContent && + Config.test(OnDiskOutputConfig::ForwardWriteThroughBuffer) && + BufferedContent->size() >= + OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer && + (BufferedContent->size() & (PageSize - 1)) != 0) { + // Synchronize the mapping to disk. + Mapping->sync(); + + // Forward the buffer. + *MaybeContent = + ContentBuffer(convertMappedFileToBuffer(std::move(Mapping))); + } + // Else, Mapping to destroy itself. + } else if (OS) { + // Write out the content and close the stream. + assert(BufferedContent && "Need to write content to a stream"); + OS->write(BufferedContent->begin(), BufferedContent->size()); + OS = nullptr; + } else { + assert(!BufferedContent && + "Content in memory with no way to write it to disk?"); + } + + // Return early if there's no temporary path. + if (!TempPath) + return Error::success(); + + // Move temporary to the final output path. + std::error_code EC = sys::fs::rename(*TempPath, OutputPath); + if (!EC) + return Error::success(); + (void)sys::fs::remove(*TempPath); + return createOnDiskOutputRenameTempError(*TempPath, OutputPath, EC); +} + +OnDiskOutputDestination::~OnDiskOutputDestination() { + if (!IsOpen) + return; + + // If there's no FD we haven't created a file. + if (!FD) + return; + + // If there's a temporary file, that's the one to delete. + if (TempPath) + sys::fs::remove(*TempPath); + else + sys::fs::remove(OutputPath); +} + +Expected> +OnDiskOutputBackend::createDestination( + StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) { + auto Dest = std::make_unique(OutputPath, Config, + std::move(NextDest)); + if (Error E = Dest->initializeFile()) + return std::move(E); + return Dest; +} + +namespace { +class InMemoryOutputDestination : public OutputDestination { +public: + bool shouldStoredBufferBeOwned() const { + return Backend->shouldStoredBufferBeOwned(); + } + + Error storeContentImpl(ContentBuffer &Content) override; + + InMemoryOutputDestination(IntrusiveRefCntPtr Backend, + std::unique_ptr Next) + : OutputDestination(std::move(Next)), Backend(std::move(Backend)) {} + +protected: + IntrusiveRefCntPtr Backend; +}; +} // anonymous namespace + +void InMemoryOutputBackend::anchor() {} + +Error InMemoryOutputDestination::storeContentImpl(ContentBuffer &Content) { + assert(Content.getBytes().end()[0] == 0 && "Expected null-terminated buffer"); + InMemoryFileSystem &FS = Backend->getInMemoryFS(); + + // Check the content of any existing buffer. We cannot rely on the logic in + // InMemoryFileSystem::addFile, since Content and/or other backends could + // have dangling references to the buffer and addFile will destroy it if it + // can't be added. + // + // FIXME: Add an API to InMemoryFileSystem to get out an + // Optional to avoid an unnecessary malloc of MemoryBuffer. + StringRef OutputPath = Content.getPath(); + if (ErrorOr> ExistingBuffer = + FS.getBufferForFile(OutputPath)) + return ExistingBuffer->get()->getBuffer() == Content.getBytes() + ? Error::success() + : createCannotOverwriteExistingOutputError(OutputPath); + + std::unique_ptr Buffer = shouldStoredBufferBeOwned() + ? Content.takeOwnedBufferOrCopy() + : Content.takeBuffer(); + bool WasAdded = FS.addFile(OutputPath, 0, std::move(Buffer)); + assert(WasAdded && "Already checked this would work; what changed?"); + return Error::success(); +} + +Expected> +InMemoryOutputBackend::createDestination( + StringRef OutputPath, OutputConfig, + std::unique_ptr NextDest) { + if (shouldRejectExistingFiles() && InMemoryFS->exists(OutputPath)) + return createCannotOverwriteExistingOutputError(OutputPath); + return std::make_unique(this, std::move(NextDest)); +} + +namespace { +class MirroringOutputBackend : public OutputBackend { +public: + Expected> + createDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) override; + + MirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) + : Backend1(Backend1), Backend2(Backend2) {} + +private: + IntrusiveRefCntPtr Backend1; + IntrusiveRefCntPtr Backend2; +}; +} // anonymous namespace + +IntrusiveRefCntPtr llvm::vfs::makeMirroringOutputBackend( + IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) { + return std::make_unique(std::move(Backend1), + std::move(Backend2)); +} + +Expected> +MirroringOutputBackend::createDestination( + StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) { + // Visit in reverse order so that the first backend gets the content before + // the second one. + for (OutputBackend *B : {&*Backend2, &*Backend1}) { + Expected> Dest = + B->createDestination(OutputPath, Config, std::move(NextDest)); + if (!Dest) + return Dest.takeError(); + NextDest = std::move(*Dest); + } + + return std::move(NextDest); +} + +namespace { +class FilteringOutputBackend : public OutputBackend { +public: + Expected> + createDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) override { + if (!Filter(OutputPath, Config)) + return std::move(NextDest); + return UnderlyingBackend->createDestination(OutputPath, Config, + std::move(NextDest)); + } + + using FilterType = unique_function; + FilteringOutputBackend(IntrusiveRefCntPtr UnderlyingBackend, + FilterType Filter) + : UnderlyingBackend(std::move(UnderlyingBackend)), + Filter(std::move(Filter)) {} + +private: + IntrusiveRefCntPtr UnderlyingBackend; + FilterType Filter; +}; +} // anonymous namespace + +IntrusiveRefCntPtr llvm::vfs::makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + FilteringOutputBackend::FilterType Filter) { + return std::make_unique(std::move(UnderlyingBackend), + std::move(Filter)); +} Index: llvm/unittests/Support/CMakeLists.txt =================================================================== --- llvm/unittests/Support/CMakeLists.txt +++ llvm/unittests/Support/CMakeLists.txt @@ -58,6 +58,8 @@ MemoryTest.cpp NativeFormatTests.cpp OptimizedStructLayoutTest.cpp + OutputConfigTest.cpp + OutputManagerTest.cpp ParallelTest.cpp Path.cpp ProcessTest.cpp Index: llvm/unittests/Support/OutputConfigTest.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/OutputConfigTest.cpp @@ -0,0 +1,468 @@ +//===- unittests/Support/OutputConfigTest.cpp - OutputConfig tests --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/OutputManager.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +TEST(PartialOutputConfigTest, construct) { + EXPECT_TRUE(PartialOutputConfig().empty()); +} + +TEST(PartialOutputConfigTest, set) { + constexpr ClientIntentOutputConfig ClientIntentValue = + ClientIntentOutputConfig::NeedsSeeking; + constexpr OnDiskOutputConfig OnDiskValue = OnDiskOutputConfig::UseTemporary; + static_assert(unsigned(ClientIntentValue) == unsigned(OnDiskValue), + "Coverage depends on having the same value representation"); + + { + auto ClientIntent = PartialOutputConfig().set(ClientIntentValue); + EXPECT_FALSE(ClientIntent.empty()); + EXPECT_EQ(true, ClientIntent.test(ClientIntentValue)); + EXPECT_EQ(None, ClientIntent.test(OnDiskValue)); + } + + { + auto OnDisk = PartialOutputConfig().set(OnDiskValue); + EXPECT_FALSE(OnDisk.empty()); + EXPECT_EQ(true, OnDisk.test(OnDiskValue)); + EXPECT_EQ(None, OnDisk.test(ClientIntentValue)); + } + + { + auto ClientIntentNotOnDisk = + PartialOutputConfig().set(ClientIntentValue).set(OnDiskValue, false); + EXPECT_EQ(true, ClientIntentNotOnDisk.test(ClientIntentValue)); + EXPECT_EQ(false, ClientIntentNotOnDisk.test(OnDiskValue)); + } + + { + auto BothSetToTrue = + PartialOutputConfig().set({ClientIntentValue, OnDiskValue}); + EXPECT_EQ(true, BothSetToTrue.test(ClientIntentValue)); + EXPECT_EQ(true, BothSetToTrue.test(OnDiskValue)); + } + + { + auto BothSetToFalse = + PartialOutputConfig().set({ClientIntentValue, OnDiskValue}, false); + EXPECT_EQ(false, BothSetToFalse.test(ClientIntentValue)); + EXPECT_EQ(false, BothSetToFalse.test(OnDiskValue)); + } +} + +TEST(PartialOutputConfigTest, reset) { + { + auto NotClientIntent = + PartialOutputConfig().reset(ClientIntentOutputConfig::NeedsSeeking); + EXPECT_FALSE(NotClientIntent.empty()); + EXPECT_EQ(false, + NotClientIntent.test(ClientIntentOutputConfig::NeedsSeeking)); + } + + { + auto BothDisabled = + PartialOutputConfig().reset({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}); + EXPECT_FALSE(BothDisabled.empty()); + EXPECT_EQ(false, BothDisabled.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(false, BothDisabled.test(ClientIntentOutputConfig::NeedsSeeking)); + } + + { + auto ResetAfterSet = PartialOutputConfig() + .set(ClientIntentOutputConfig::NeedsSeeking) + .reset(ClientIntentOutputConfig::NeedsSeeking); + EXPECT_EQ(false, + ResetAfterSet.test(ClientIntentOutputConfig::NeedsSeeking)); + } + + { + auto ResetAfterSetBoth = PartialOutputConfig() + .set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .reset({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}); + EXPECT_EQ(false, + ResetAfterSetBoth.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(false, ResetAfterSetBoth.test(OnDiskOutputConfig::UseTemporary)); + } + + { + auto SetAfterReset = PartialOutputConfig() + .reset(ClientIntentOutputConfig::NeedsSeeking) + .set(ClientIntentOutputConfig::NeedsSeeking); + EXPECT_EQ(true, SetAfterReset.test(ClientIntentOutputConfig::NeedsSeeking)); + } + + { + auto SetAfterResetBoth = PartialOutputConfig() + .reset({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}); + EXPECT_EQ(true, + SetAfterResetBoth.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, SetAfterResetBoth.test(OnDiskOutputConfig::UseTemporary)); + } +} + +TEST(PartialOutputConfigTest, drop) { + { + auto DropClientIntent = PartialOutputConfig() + .set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .drop(ClientIntentOutputConfig::NeedsSeeking); + EXPECT_FALSE(DropClientIntent.empty()); + EXPECT_EQ(None, + DropClientIntent.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, DropClientIntent.test(OnDiskOutputConfig::UseTemporary)); + } + + { + auto DropClientIntentAfterReset = + PartialOutputConfig() + .reset({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .drop(ClientIntentOutputConfig::NeedsSeeking); + EXPECT_FALSE(DropClientIntentAfterReset.empty()); + EXPECT_EQ(None, DropClientIntentAfterReset.test( + ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(false, DropClientIntentAfterReset.test( + OnDiskOutputConfig::UseTemporary)); + } + + { + auto DropOnDisk = PartialOutputConfig() + .set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .drop(OnDiskOutputConfig::UseTemporary); + EXPECT_FALSE(DropOnDisk.empty()); + EXPECT_EQ(true, DropOnDisk.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(None, DropOnDisk.test(OnDiskOutputConfig::UseTemporary)); + } + + { + auto DropBoth = PartialOutputConfig() + .set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .drop({OnDiskOutputConfig::UseTemporary, + ClientIntentOutputConfig::NeedsSeeking}); + EXPECT_TRUE(DropBoth.empty()); + EXPECT_EQ(None, DropBoth.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(None, DropBoth.test(OnDiskOutputConfig::UseTemporary)); + } + + { + auto DropBothAfterReset = + PartialOutputConfig() + .reset({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}) + .drop({OnDiskOutputConfig::UseTemporary, + ClientIntentOutputConfig::NeedsSeeking}); + EXPECT_TRUE(DropBothAfterReset.empty()); + EXPECT_EQ(None, + DropBothAfterReset.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(None, DropBothAfterReset.test(OnDiskOutputConfig::UseTemporary)); + } +} + +TEST(PartialOutputConfigTest, applyOverrides) { + auto getConfig = []() { + PartialOutputConfig Config; + Config.set(ClientIntentOutputConfig::NeedsSeeking); + Config.set(OnDiskOutputConfig::UseTemporary); + Config.reset(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories); + Config.reset(OnDiskOutputConfig::RemoveFileOnSignal); + return Config; + }; + + { + // No overrides. + PartialOutputConfig Config = getConfig(), Overrides; + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(None, Config.test(OnDiskOutputConfig::OpenFlagText)); + } + + { + // No config, just overrides. + PartialOutputConfig Overrides = getConfig(), Config; + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(None, Config.test(OnDiskOutputConfig::OpenFlagText)); + } + + { + // Some the same, some different. + PartialOutputConfig Config = getConfig(), Overrides; + Overrides.set(ClientIntentOutputConfig::NeedsSeeking); + Overrides.reset(OnDiskOutputConfig::UseTemporary); + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(None, Config.test(OnDiskOutputConfig::OpenFlagText)); + } + + { + // New settings. + PartialOutputConfig Config = getConfig(), Overrides; + Overrides.set(ClientIntentOutputConfig::NeedsReadAccess); + Overrides.reset(OnDiskOutputConfig::OpenFlagText); + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsReadAccess)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::OpenFlagText)); + } +} + +TEST(PartialOutputConfigTest, applyDefaults) { + auto getConfig = []() { + PartialOutputConfig Config; + Config.set(ClientIntentOutputConfig::NeedsSeeking); + Config.set(OnDiskOutputConfig::UseTemporary); + Config.reset(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories); + Config.reset(OnDiskOutputConfig::RemoveFileOnSignal); + return Config; + }; + + { + // No defaults. + PartialOutputConfig Config = getConfig(), Defaults; + Config.applyDefaults(Defaults); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(None, Config.test(OnDiskOutputConfig::OpenFlagText)); + } + + { + // No config, just defaults. + PartialOutputConfig Defaults = getConfig(), Config; + Config.applyDefaults(Defaults); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(None, Config.test(OnDiskOutputConfig::OpenFlagText)); + } + + { + // Some the same, some different. + PartialOutputConfig Config = getConfig(), Defaults; + Defaults.set(ClientIntentOutputConfig::NeedsSeeking); + Defaults.reset(OnDiskOutputConfig::UseTemporary); + Config.applyDefaults(Defaults); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(None, Config.test(OnDiskOutputConfig::OpenFlagText)); + } + + { + // New settings. + PartialOutputConfig Config = getConfig(), Overrides; + Overrides.set(ClientIntentOutputConfig::NeedsReadAccess); + Overrides.reset(OnDiskOutputConfig::OpenFlagText); + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsReadAccess)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::OpenFlagText)); + } +} + +TEST(OutputConfigTest, construct) { + EXPECT_TRUE(OutputConfig().none()); + EXPECT_FALSE(OutputConfig().test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_FALSE(OutputConfig().test(OnDiskOutputConfig::UseTemporary)); + + { + OutputConfig Config = {ClientIntentOutputConfig::NeedsSeeking}; + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_TRUE(Config.test(ClientIntentOutputConfig::NeedsSeeking)); + } + + { + OutputConfig Config = {ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary}; + EXPECT_FALSE(Config.none()); + EXPECT_TRUE(Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_TRUE(Config.test(ClientIntentOutputConfig::NeedsSeeking)); + } +} + +TEST(OutputConfigTest, set) { + { + // Set one. + OutputConfig Config; + Config.set(ClientIntentOutputConfig::NeedsSeeking); + EXPECT_FALSE(Config.none()); + EXPECT_TRUE(Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_FALSE(Config.test(OnDiskOutputConfig::UseTemporary)); + + // Undo it. + Config.set(ClientIntentOutputConfig::NeedsSeeking, false); + EXPECT_TRUE(Config.none()); + EXPECT_FALSE(Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_FALSE(Config.test(OnDiskOutputConfig::UseTemporary)); + } + + { + // Set multiple. + OutputConfig Config; + Config.set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary, + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories}); + EXPECT_FALSE(Config.none()); + EXPECT_TRUE(Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_TRUE(Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_TRUE( + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + + // Undo two of them. + Config.set({ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories}, + false); + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_FALSE( + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_TRUE(Config.test(OnDiskOutputConfig::UseTemporary)); + } +} + +TEST(OutputConfigTest, reset) { + constexpr ClientIntentOutputConfig ClientIntentValue = + ClientIntentOutputConfig::NeedsSeeking; + constexpr OnDiskOutputConfig OnDiskValue = OnDiskOutputConfig::UseTemporary; + static_assert(unsigned(ClientIntentValue) == unsigned(OnDiskValue), + "Coverage depends on having the same value representation"); + + { + // Reset one. + OutputConfig Config = {ClientIntentValue, OnDiskValue}; + Config.reset(ClientIntentValue); + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(ClientIntentValue)); + EXPECT_TRUE(Config.test(OnDiskValue)); + } + + { + // Reset both. + OutputConfig Config = {ClientIntentValue, OnDiskValue}; + Config.reset({ClientIntentValue, OnDiskValue}); + EXPECT_TRUE(Config.none()); + EXPECT_FALSE(Config.test(ClientIntentValue)); + EXPECT_FALSE(Config.test(OnDiskValue)); + } + + { + // Reset multiple (but not all). + OutputConfig Config = { + ClientIntentValue, OnDiskValue, + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories}; + Config.reset({ClientIntentValue, + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories}); + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(ClientIntentValue)); + EXPECT_FALSE( + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_TRUE(Config.test(OnDiskValue)); + } +} + +TEST(OutputConfigTest, applyOverrides) { + OutputConfig InitialConfig = { + ClientIntentOutputConfig::NeedsSeeking, + OnDiskOutputConfig::UseTemporary, + }; + + { + // No overrides. + OutputConfig Config = InitialConfig; + PartialOutputConfig Overrides; + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + } + + { + // Some the same, some different. + OutputConfig Config = InitialConfig; + PartialOutputConfig Overrides; + Overrides.set(ClientIntentOutputConfig::NeedsSeeking); + Overrides.reset(OnDiskOutputConfig::UseTemporary); + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::OpenFlagText)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + } + + { + // New settings. + OutputConfig Config = InitialConfig; + PartialOutputConfig Overrides; + Overrides.set(ClientIntentOutputConfig::NeedsReadAccess); + Overrides.reset(OnDiskOutputConfig::OpenFlagText); + Config.applyOverrides(Overrides); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_EQ(true, Config.test(ClientIntentOutputConfig::NeedsReadAccess)); + EXPECT_EQ(true, Config.test(OnDiskOutputConfig::UseTemporary)); + EXPECT_EQ( + false, + Config.test(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::RemoveFileOnSignal)); + EXPECT_EQ(false, Config.test(OnDiskOutputConfig::OpenFlagText)); + } +} + +} // anonymous namespace Index: llvm/unittests/Support/OutputManagerTest.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/OutputManagerTest.cpp @@ -0,0 +1,1237 @@ +//===- unittests/Support/OutputManagerTest.cpp - OutputManager tests ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/OutputManager.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +using ContentBuffer = OutputDestination::ContentBuffer; + +TEST(OutputDestinationContentBufferTest, takeBuffer) { + MemoryBufferRef B("data", "name"); + + ContentBuffer Content = B; + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(B.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeBuffer(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + + // The buffer should be a reference to the original data, and Content should + // still have it too. + EXPECT_EQ(B.getBufferStart(), Taken->getBufferStart()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); +} + +TEST(OutputDestinationContentBufferTest, takeBufferVector) { + SmallString<0> V = StringRef("data"); + StringRef R = V; + + ContentBuffer Content(std::move(V), "name"); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(R.begin(), Content.getBytes().begin()); + EXPECT_NE(V.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeBuffer(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.begin(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); +} + +TEST(OutputDestinationContentBufferTest, takeBufferVectorSmallStorage) { + // Something using small storage will have to move. + SmallString<128> V = StringRef("data"); + ASSERT_EQ(128u, V.capacity()); + StringRef R = V; + + ContentBuffer Content(std::move(V), "name"); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_NE(R.begin(), Content.getBytes().begin()); +} + +TEST(OutputDestinationContentBufferTest, takeBufferMemoryBuffer) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data", "name"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeBuffer(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrNull) { + MemoryBufferRef B("data", "name"); + + ContentBuffer Content = B; + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(B.getBufferStart(), Content.getBytes().begin()); + + // ContentBuffer doesn't own this. + EXPECT_FALSE(Content.takeOwnedBufferOrNull()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrNullVector) { + SmallString<0> V = StringRef("data"); + StringRef R = V; + + ContentBuffer Content(std::move(V), "name"); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(R.begin(), Content.getBytes().begin()); + EXPECT_NE(V.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrNull(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.begin(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + + // The next call should fail since it's no longer owned. + EXPECT_FALSE(Content.takeOwnedBufferOrNull()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrNullVectorEmpty) { + SmallString<0> V; + + ContentBuffer Content(std::move(V), "name"); + EXPECT_EQ("", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + + // Check that ContentBuffer::takeOwnedBufferOrNull() fails for an empty + // vector. + EXPECT_FALSE(Content.ownsContent()); + ASSERT_FALSE(Content.takeOwnedBufferOrNull()); + + // ContentBuffer::takeBuffer() should still work. + std::unique_ptr Taken = Content.takeBuffer(); + ASSERT_TRUE(Taken); + EXPECT_EQ("", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrNullMemoryBuffer) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data", "name"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrNull(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + + // The next call should fail since it's no longer owned. + EXPECT_FALSE(Content.takeOwnedBufferOrNull()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrCopy) { + MemoryBufferRef B("data", "name"); + + ContentBuffer Content = B; + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(B.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrCopy(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + + // We should have a copy, but Content should be unchanged. + EXPECT_NE(B.getBufferStart(), Taken->getBufferStart()); + EXPECT_EQ(B.getBufferStart(), Content.getBytes().begin()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrCopyVector) { + SmallString<0> V = StringRef("data"); + StringRef R = V; + + ContentBuffer Content(std::move(V), "name"); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(R.begin(), Content.getBytes().begin()); + EXPECT_NE(V.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrCopy(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + + // Content should still have a reference to the data, but it's no longer + // owned. + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + EXPECT_FALSE(Content.takeOwnedBufferOrNull()); + + // The next call should get a different (but equal) buffer. + std::unique_ptr Copy = Content.takeOwnedBufferOrCopy(); + ASSERT_TRUE(Copy); + EXPECT_EQ("data", Copy->getBuffer()); + EXPECT_EQ("name", Copy->getBufferIdentifier()); + EXPECT_NE(Taken->getBufferStart(), Copy->getBufferStart()); +} + +TEST(OutputDestinationContentBufferTest, takeOwnedBufferOrCopyMemoryBuffer) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data", "name"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ("name", Content.getPath()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrCopy(); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + + // The next call should get a different (but equal) buffer. + std::unique_ptr Copy = Content.takeOwnedBufferOrCopy(); + ASSERT_TRUE(Copy); + EXPECT_EQ("data", Copy->getBuffer()); + EXPECT_EQ("name", Copy->getBufferIdentifier()); + EXPECT_NE(Taken->getBufferStart(), Copy->getBufferStart()); +} + +struct OnDiskTempDirectory { + bool Created = false; + SmallString<128> Path; + + OnDiskTempDirectory() = delete; + explicit OnDiskTempDirectory(StringRef Name) { + if (!sys::fs::createUniqueDirectory(Name, Path)) + Created = true; + } + ~OnDiskTempDirectory() { + if (Created) + sys::fs::remove_directories(Path); + } +}; + +struct OnDiskFile { + const OnDiskTempDirectory &D; + SmallString<128> Path; + StringRef ParentPath; + StringRef Filename; + StringRef Stem; + StringRef Extension; + std::unique_ptr LastBuffer; + + template + OnDiskFile(const OnDiskTempDirectory &D, PathArgTypes &&... PathArgs) : D(D) { + assert(D.Created); + sys::path::append(Path, D.Path, std::forward(PathArgs)...); + ParentPath = sys::path::parent_path(Path); + Filename = sys::path::filename(Path); + Stem = sys::path::stem(Filename); + Extension = sys::path::extension(Filename); + } + + Optional findTemp() const { + std::error_code EC; + for (sys::fs::directory_iterator I(ParentPath, EC), E; !EC && I != E; + I.increment(EC)) { + StringRef TempPath = I->path(); + if (!TempPath.startswith(D.Path)) + continue; + + // Look for "-*..tmp". + if (sys::path::extension(TempPath) != ".tmp") + continue; + + // Drop the ".tmp" and check the extension and stem. + StringRef TempStem = sys::path::stem(TempPath); + if (sys::path::extension(TempStem) != Extension) + continue; + StringRef OriginalStem = sys::path::stem(TempStem); + if (!OriginalStem.startswith(Stem)) + continue; + if (!OriginalStem.drop_front(Stem.size()).startswith("-")) + continue; + + // Found it. + return OnDiskFile(D, TempPath.drop_front(D.Path.size() + 1)); + } + return None; + } + + Optional getCurrentUniqueID() { + sys::fs::file_status Status; + sys::fs::status(Path, Status, /*follow=*/false); + if (!sys::fs::is_regular_file(Status)) + return None; + return Status.getUniqueID(); + } + + bool hasUniqueID(sys::fs::UniqueID ID) { + auto CurrentID = getCurrentUniqueID(); + if (!CurrentID) + return false; + return *CurrentID == ID; + } + + Optional getCurrentContent() { + auto OnDiskOrErr = MemoryBuffer::getFile(Path); + if (!OnDiskOrErr) + return None; + LastBuffer = std::move(*OnDiskOrErr); + return LastBuffer->getBuffer(); + } + + bool equalsCurrentContent(StringRef Data) { + auto CurrentContent = getCurrentContent(); + if (!CurrentContent) + return false; + return *CurrentContent == Data; + } + + bool equalsCurrentContent(NoneType) { return getCurrentContent() == None; } +}; + +struct OutputManagerTest : public ::testing::Test { + Optional D; + + void TearDown() override { D = None; } + + bool makeTempDirectory() { + D.emplace("OutputManagerTest.d"); + return D->Created; + } + + template + std::unique_ptr expectedToPointer(Expected> Expected) { + if (auto Maybe = expectedToOptional(std::move(Expected))) + return std::move(*Maybe); + return nullptr; + } + + std::unique_ptr openBufferForRead(InMemoryFileSystem &FS, + StringRef Path) { + if (auto FileOrErr = FS.openFileForRead(Path)) + if (*FileOrErr) + if (auto BufferOrErr = (*FileOrErr)->getBuffer(Path)) + if (*BufferOrErr) + return std::move(*BufferOrErr); + return nullptr; + } +}; + +TEST_F(OutputManagerTest, DefaultConfig) { + OutputManager M; + + // Check some important settings in the default configuration. + EXPECT_TRUE(M.getDefaults().test(ClientIntentOutputConfig::NeedsSeeking)); + EXPECT_TRUE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + EXPECT_FALSE(M.getDefaults().test( + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories)); + EXPECT_TRUE(M.getDefaults().test(OnDiskOutputConfig::RemoveFileOnSignal)); +} + +TEST_F(OutputManagerTest, Null) { + OutputManager M(makeNullOutputBackend()); + + // Create the output. + auto O = expectedToPointer(M.createOutput("ignored.data")); + ASSERT_TRUE(O); + ASSERT_TRUE(O->getOS()); + + // Write some data into it and close the stream. + *O->getOS() << "some data"; + (void)O->takeOS(); + + // Closing should have no effect. + EXPECT_FALSE(errorToBool(O->close())); +} + +TEST_F(OutputManagerTest, NullErase) { + OutputManager M(makeNullOutputBackend()); + + // Create the output. + auto O = expectedToPointer(M.createOutput("ignored.data")); + ASSERT_TRUE(O); + ASSERT_TRUE(O->getOS()); + + // Write some data into it and close the stream. + *O->takeOS() << "some data"; + + // Erasing should have no effect. + O->erase(); +} + +TEST_F(OutputManagerTest, OnDisk) { + OutputManager M; + ASSERT_TRUE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + Optional TempID = Temp->getCurrentUniqueID(); + ASSERT_TRUE(TempID); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(Temp->equalsCurrentContent("some data")); + EXPECT_TRUE(File.equalsCurrentContent(None)); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // The temp file should have been moved. + EXPECT_TRUE(File.hasUniqueID(*TempID)); +} + +TEST_F(OutputManagerTest, OnDiskErase) { + OutputManager M; + ASSERT_TRUE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + + // Write some data into it and close the stream. + *O->takeOS() << "some data"; + + // Erase and check that the temp is gone. + O->erase(); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent(None)); +} + +TEST_F(OutputManagerTest, OnDiskCreateMissingDirectories) { + OutputManager M; + ASSERT_TRUE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + M.getDefaults().set(OnDiskOutputConfig::UseTemporaryCreateMissingDirectories); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "missing", "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + Optional TempID = Temp->getCurrentUniqueID(); + ASSERT_TRUE(TempID); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(Temp->equalsCurrentContent("some data")); + EXPECT_TRUE(File.equalsCurrentContent(None)); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // The temp file should have been moved. + EXPECT_TRUE(File.hasUniqueID(*TempID)); +} + +TEST_F(OutputManagerTest, OnDiskNoCreateMissingDirectories) { + OutputManager M; + ASSERT_TRUE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + M.getDefaults().reset( + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "missing", "on-disk.data"); + + // Fail to create the missing directory. + auto Expected = M.createOutput(File.Path); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value()); +} + +TEST_F(OutputManagerTest, OnDiskNoTemporary) { + OutputManager M; + M.getDefaults().reset(OnDiskOutputConfig::UseTemporary); + ASSERT_FALSE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // File should exist with no temporary. + EXPECT_TRUE(File.equalsCurrentContent("")); + Optional ID = File.getCurrentUniqueID(); + ASSERT_TRUE(ID); + Optional Temp = File.findTemp(); + ASSERT_FALSE(Temp); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + EXPECT_TRUE(File.hasUniqueID(*ID)); +} + +TEST_F(OutputManagerTest, OnDiskNoTemporaryErase) { + OutputManager M; + M.getDefaults().reset(OnDiskOutputConfig::UseTemporary); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // File should exist with no temporary. + EXPECT_TRUE(File.equalsCurrentContent("")); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // Erase. Since UseTemporary is off the file should now be gone. + O->erase(); + EXPECT_TRUE(File.equalsCurrentContent(None)); +} + +TEST_F(OutputManagerTest, OnDiskWriteThroughBuffer) { + OutputManager M; + ASSERT_TRUE(M.getDefaults().test(OnDiskOutputConfig::UseTemporary)); + M.getDefaults().set(OnDiskOutputConfig::UseWriteThroughBuffer); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + Optional TempID = Temp->getCurrentUniqueID(); + ASSERT_TRUE(TempID); + + // Write some data into it and flush. Confirm it's not on disk because it's + // waiting to be written using a write-through buffer. + *O->takeOS() << "some data"; + EXPECT_TRUE(Temp->equalsCurrentContent("")); + EXPECT_TRUE(File.equalsCurrentContent(None)); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + EXPECT_TRUE(File.hasUniqueID(*TempID)); +} + +struct CaptureLastBufferOutputBackend : public OutputBackend { + struct Destination : public OutputDestination { + Error storeContentImpl(ContentBuffer &Content) final { + Storage = Content.takeBuffer(); + return Error::success(); + } + + Destination(std::unique_ptr &Storage, + std::unique_ptr NextDest) + : OutputDestination(std::move(NextDest)), Storage(Storage) {} + + std::unique_ptr &Storage; + }; + + Expected> + createDestination(StringRef OutputPath, OutputConfig Config, + std::unique_ptr NextDest) final { + return std::make_unique(Storage, std::move(NextDest)); + } + + std::unique_ptr &Storage; + CaptureLastBufferOutputBackend(std::unique_ptr &Storage) + : Storage(Storage) {} +}; + +TEST_F(OutputManagerTest, OnDiskWriteThroughBufferReturned) { + std::unique_ptr Buffer; + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(Buffer))); + M.getDefaults().set({OnDiskOutputConfig::UseWriteThroughBuffer, + OnDiskOutputConfig::ForwardWriteThroughBuffer}); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write enough data to get a write-through buffer, and add one to ensure + // it's not page-aligned. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer + 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Check that the saved buffer is a write-through buffer. + EXPECT_TRUE(Buffer); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); + EXPECT_EQ(MemoryBuffer::MemoryBuffer_MMap, Buffer->getBufferKind()); +} + +TEST_F(OutputManagerTest, OnDiskWriteThroughBufferNotReturned) { + std::unique_ptr Buffer; + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(Buffer))); + M.getDefaults().set({OnDiskOutputConfig::UseWriteThroughBuffer, + OnDiskOutputConfig::ForwardWriteThroughBuffer}); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write almost enough data to return write-through buffer, but not quite. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer - 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Check that the saved buffer is not write-through buffer. + EXPECT_TRUE(Buffer); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); + EXPECT_EQ(MemoryBuffer::MemoryBuffer_Malloc, Buffer->getBufferKind()); +} + +TEST_F(OutputManagerTest, FilteredOnDisk) { + OutputManager M( + makeFilteringOutputBackend(makeIntrusiveRefCnt(), + [](StringRef, OutputConfig) { return true; })); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. The content should be there. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent("some data")); +} + +TEST_F(OutputManagerTest, FilteredOnDiskSkipped) { + OutputManager M(makeFilteringOutputBackend( + makeIntrusiveRefCnt(), + [](StringRef, OutputConfig) { return false; })); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. It should not exist on disk. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(None)); +} + +TEST_F(OutputManagerTest, InMemory) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeIntrusiveRefCnt(FS)); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + ASSERT_FALSE(FS->exists(Path)); + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + ASSERT_FALSE(FS->exists(Path)); + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS->exists(Path)); + std::unique_ptr Buffer = openBufferForRead(*FS, Path); + EXPECT_EQ(Path, Buffer->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer->getBuffer()); +} + +TEST_F(OutputManagerTest, InMemoryRejectExistingFiles) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeIntrusiveRefCnt(FS); + OutputManager M(Backend); + + // Add a conflicting file to FS. + StringRef Path = "//root/in/memory.data"; + FS->addFile(Path, 0, MemoryBuffer::getMemBuffer("some data")); + + // Check that we get the error from failing to clobber the in-memory file. + Backend->setShouldRejectExistingFiles(true); + auto Expected = M.createOutput(Path); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::device_or_resource_busy), EC.value()); + + // No error if we turn off the flag though. + Backend->setShouldRejectExistingFiles(false); + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Writing identical data should be okay. + *O->takeOS() << "some data"; + ASSERT_FALSE(errorToBool(O->close())); +} + +TEST_F(OutputManagerTest, InMemoryMismatchedContent) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeIntrusiveRefCnt(FS)); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + ASSERT_FALSE(FS->exists(Path)); + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_FALSE(FS->exists(Path)); + ASSERT_TRUE(O); + + // Add a conflicting file to FS. + FS->addFile(Path, 0, MemoryBuffer::getMemBuffer("old data")); + ASSERT_TRUE(FS->exists(Path)); + + // Write some data into it, flush, and close. + *O->takeOS() << "new data"; + Error E = O->close(); + + // Check the error. + ASSERT_TRUE(bool(E)); + std::error_code EC = errorToErrorCode(std::move(E)); + EXPECT_EQ(int(std::errc::device_or_resource_busy), EC.value()); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemory) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS))); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // Lookup the file in FS. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer->getBuffer()); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemoryRejectExistingFiles) { + auto FS = makeIntrusiveRefCnt(); + auto InMemoryBackend = makeIntrusiveRefCnt(FS); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), InMemoryBackend)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Add a file with the same name to FS. + FS->addFile(File.Path, 0, MemoryBuffer::getMemBuffer("some data")); + + // Check that we get the error from failing to clobber the in-memory file. + InMemoryBackend->setShouldRejectExistingFiles(true); + auto Expected = M.createOutput(File.Path); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::device_or_resource_busy), EC.value()); + + // No error if we turn off the flag though. + InMemoryBackend->setShouldRejectExistingFiles(false); + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Writing identical data should be okay. + *O->takeOS() << "some data"; + ASSERT_FALSE(errorToBool(O->close())); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemoryMismatchedContent) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS))); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the output. + ASSERT_FALSE(FS->exists(File.Path)); + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_FALSE(FS->exists(File.Path)); + ASSERT_TRUE(O); + + // Add a conflicting file to FS. + FS->addFile(File.Path, 0, MemoryBuffer::getMemBuffer("old data")); + ASSERT_TRUE(FS->exists(File.Path)); + + // Write some data into it, flush, and close. + *O->takeOS() << "new data"; + Error E = O->close(); + + // Check the error. + ASSERT_TRUE(bool(E)); + std::error_code EC = errorToErrorCode(std::move(E)); + EXPECT_EQ(int(std::errc::device_or_resource_busy), EC.value()); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemoryNoCreateMissingDirectories) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS))); + M.getDefaults().reset( + OnDiskOutputConfig::UseTemporaryCreateMissingDirectories); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "missing", "on-disk.data"); + + // Check that we get the error from failing to create the missing directory + // on-disk. + auto Expected = M.createOutput(File.Path); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value()); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemoryWriteThrough) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS))); + M.getDefaults().set({OnDiskOutputConfig::UseWriteThroughBuffer, + OnDiskOutputConfig::ForwardWriteThroughBuffer}); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write enough data to get a write-through buffer, and add one to ensure + // it's not page-aligned. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer + 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Lookup the file in FS. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemoryWriteThroughPageAligned) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS))); + M.getDefaults().set({OnDiskOutputConfig::UseWriteThroughBuffer, + OnDiskOutputConfig::ForwardWriteThroughBuffer}); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write just enough data to get a write-through buffer. It'll likely be + // page-aligned. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer, 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Lookup the file in FS. Ensure no errors when it creates a null-terminated + // reference to the buffer. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); +} + +TEST_F(OutputManagerTest, MirrorOnDiskInMemoryWriteThroughTooSmall) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS))); + M.getDefaults().set({OnDiskOutputConfig::UseWriteThroughBuffer, + OnDiskOutputConfig::ForwardWriteThroughBuffer}); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(M.createOutput(File.Path)); + ASSERT_TRUE(O); + + // Write almost (but not quite) enough data to get a write-through buffer. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer - 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Lookup the file in FS. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); +} + +TEST_F(OutputManagerTest, FilteredInMemory) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M( + makeFilteringOutputBackend(makeIntrusiveRefCnt(FS), + [](StringRef, OutputConfig) { return true; })); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS->exists(Path)); + std::unique_ptr Buffer = openBufferForRead(*FS, Path); + EXPECT_EQ(Path, Buffer->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer->getBuffer()); +} + +TEST_F(OutputManagerTest, FilteredInMemorySkipped) { + auto FS = makeIntrusiveRefCnt(); + OutputManager M(makeFilteringOutputBackend( + makeIntrusiveRefCnt(FS), + [](StringRef, OutputConfig) { return false; })); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + ASSERT_FALSE(FS->exists(Path)); + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // The file should have been filtered out. + ASSERT_FALSE(FS->exists(Path)); +} + +TEST_F(OutputManagerTest, MirrorFilteredInMemoryInMemoryNotFiltered) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeFilteringOutputBackend( + makeIntrusiveRefCnt(FS1), + [](StringRef, OutputConfig) { return false; }), + makeIntrusiveRefCnt(FS2))); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. It should only be skipped in FS1. + EXPECT_TRUE(!FS1->exists(Path)); + ASSERT_TRUE(FS2->exists(Path)); + std::unique_ptr Buffer2 = openBufferForRead(*FS2, Path); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer2->getBuffer()); +} + +TEST_F(OutputManagerTest, MirrorInMemoryWithFilteredInMemory) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(FS1), + makeFilteringOutputBackend( + makeIntrusiveRefCnt(FS2), + [](StringRef, OutputConfig) { return false; }))); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. It should only be skipped in FS1. + EXPECT_TRUE(!FS2->exists(Path)); + ASSERT_TRUE(FS1->exists(Path)); + std::unique_ptr Buffer2 = openBufferForRead(*FS1, Path); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer2->getBuffer()); +} + +TEST_F(OutputManagerTest, MirrorInMemoryInMemory) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(FS1), + makeIntrusiveRefCnt(FS2))); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS1->exists(Path)); + ASSERT_TRUE(FS2->exists(Path)); + std::unique_ptr Buffer1 = openBufferForRead(*FS1, Path); + std::unique_ptr Buffer2 = openBufferForRead(*FS2, Path); + ASSERT_TRUE(Buffer1); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer1->getBufferIdentifier()); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer1->getBuffer()); + EXPECT_EQ("some data", Buffer2->getBuffer()); + + // Should be a reference to the same memory. + EXPECT_EQ(Buffer1->getBufferStart(), Buffer2->getBufferStart()); +} + +TEST_F(OutputManagerTest, MirrorInMemoryInMemoryOwnBufferForMirror) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + auto Backend2 = makeIntrusiveRefCnt(FS2); + Backend2->setShouldStoredBufferBeOwned(true); + OutputManager M(makeMirroringOutputBackend( + makeIntrusiveRefCnt(FS1), std::move(Backend2))); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(M.createOutput(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS1->exists(Path)); + ASSERT_TRUE(FS2->exists(Path)); + std::unique_ptr Buffer1 = openBufferForRead(*FS1, Path); + std::unique_ptr Buffer2 = openBufferForRead(*FS2, Path); + ASSERT_TRUE(Buffer1); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer1->getBufferIdentifier()); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer1->getBuffer()); + EXPECT_EQ("some data", Buffer2->getBuffer()); + + // Should be a copy. + EXPECT_NE(Buffer1->getBufferStart(), Buffer2->getBufferStart()); +} + +struct ScopedOutputManagerBackendTest : public OutputManagerTest {}; + +TEST_F(ScopedOutputManagerBackendTest, WritesToCorrectBackend) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + auto Backend1 = makeIntrusiveRefCnt(FS1); + auto Backend2 = makeIntrusiveRefCnt(FS2); + + // Write five files: one while both scopes are in place, before and after + // each nested scope. + StringRef BeforeOuterPath = "//root/in/memory/before-outer.data"; + StringRef BeforeInnerPath = "//root/in/memory/before-inner.data"; + StringRef MiddlePath = "//root/in/memory/middle.data"; + StringRef AfterInnerPath = "//root/in/memory/after-inner.data"; + StringRef AfterOuterPath = "//root/in/memory/after-outer.data"; + OutputManager M(Backend1); + EXPECT_EQ(Backend1.get(), &M.getBackend()); + { + std::unique_ptr BeforeOuter, BeforeInner, Middle, AfterInner, + AfterOuter; + BeforeOuter = expectedToPointer(M.createOutput(BeforeOuterPath)); + ASSERT_TRUE(BeforeOuter); + { + ScopedOutputManagerBackend Outer(M, Backend2); + EXPECT_EQ(Backend2.get(), &M.getBackend()); + BeforeInner = expectedToPointer(M.createOutput(BeforeInnerPath)); + ASSERT_TRUE(BeforeInner); + { + ScopedOutputManagerBackend Inner(M, Backend1); + EXPECT_EQ(Backend1.get(), &M.getBackend()); + Middle = expectedToPointer(M.createOutput(MiddlePath)); + ASSERT_TRUE(Middle); + } + EXPECT_EQ(Backend2.get(), &M.getBackend()); + AfterInner = expectedToPointer(M.createOutput(AfterInnerPath)); + ASSERT_TRUE(AfterInner); + } + EXPECT_EQ(Backend1.get(), &M.getBackend()); + AfterOuter = expectedToPointer(M.createOutput(AfterOuterPath)); + ASSERT_TRUE(AfterOuter); + + EXPECT_FALSE(errorToBool(AfterInner->close())); + EXPECT_FALSE(errorToBool(Middle->close())); + EXPECT_FALSE(errorToBool(BeforeInner->close())); + EXPECT_FALSE(errorToBool(AfterOuter->close())); + EXPECT_FALSE(errorToBool(BeforeOuter->close())); + } + + // Lookup the files. Middle and outer should be in FS1, while inner should be + // in FS2. + EXPECT_TRUE(FS1->exists(BeforeOuterPath)); + EXPECT_TRUE(FS1->exists(AfterOuterPath)); + EXPECT_TRUE(FS1->exists(MiddlePath)); + EXPECT_TRUE(!FS1->exists(BeforeInnerPath)); + EXPECT_TRUE(!FS1->exists(AfterInnerPath)); + EXPECT_TRUE(!FS2->exists(BeforeOuterPath)); + EXPECT_TRUE(!FS2->exists(AfterOuterPath)); + EXPECT_TRUE(!FS2->exists(MiddlePath)); + EXPECT_TRUE(FS2->exists(BeforeInnerPath)); + EXPECT_TRUE(FS2->exists(AfterInnerPath)); +} + +} // anonymous namespace