Index: llvm/include/llvm/Support/OutputBackend.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/OutputBackend.h @@ -0,0 +1,771 @@ +//===- llvm/Support/OutputBackend.h - Output management ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_OUTPUTBACKEND_H +#define LLVM_SUPPORT_OUTPUTBACKEND_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/simple_ilist.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace vfs { + +/// Set of advisory flags for backends. By default, all flags are off. +enum class OutputConfigFlag { + /// Files should be opened in text mode. Backends modelling a Windows + /// filesystem should translate newlines to carriage returns and line feeds. + /// + /// Seeking is not supported for text outputs. + Text, + + /// Advise that other processes are likely to *modify* this output after it + /// has been written (as opposed to replacing it atomically); for example, if + /// the output is a source file on-disk, an editor may modify it in place. + /// \a OnDiskOutputBackend will disable forwarding write-through buffers in + /// this case. + Volatile, + + /// No need to clean this up if there's a crash. If set, backends will not + /// install \a llvm::RemoveOnSignal or similar, even if configured to do so + /// by default. + NoCrashCleanup, + + /// This file does not need to be atomically moved into its final path. If + /// set, \a OnDiskOutputBackend will not use temporary files, even if it's + /// configured to do so by default. + NoAtomicWrite, + + /// Do not imply creation of missing directories. Backends that don't have + /// directories as a first-class concept are free to ignore this. + NoImplyCreateDirectories, + + /// Error if an output already exists and/or would be clobbered. Note: some + /// backends only know about files and directories they've created + /// themselves. + NoOverwrite, + + // Keep this last. + NumFlags, +}; + +/// Full configuration for an output for use by the \a OutputBackend. Each +/// configuration flag is either \c true or \c false. +class OutputConfig { + /// Don't use std::bitset since it's mostly not constexpr. If primitive types + /// don't have enough bits, this can use a simple constexpr-friendly bitset. + using BitsetType = unsigned char; + + /// Construct the combined enumeration from any individual enum. + constexpr static BitsetType getBitset(OutputConfigFlag Flag) { + static_assert(sizeof(BitsetType) <= sizeof(int), + "Returned literal will overflow"); + return 1u << static_cast(Flag); + } + + /// Check that the flags fit in the bitset. + static_assert(static_cast(OutputConfigFlag::NumFlags) <= + sizeof(BitsetType) * 8, + "BitsetType ran out of bits"); + +public: + /// Test whether there are no flags turned on. + constexpr bool none() const { return !Bits; } + + /// Check the value for \c Flag. + constexpr bool test(OutputConfigFlag Flag) const { + return Bits & getBitset(Flag); + } + + /// Set \c Flag to \c Value. + constexpr OutputConfig &set(OutputConfigFlag Flag, bool Value = true) { + if (Value) + Bits |= getBitset(Flag); + else + Bits &= ~getBitset(Flag); + return *this; + } + + /// Set \c Flags to \c Value. + constexpr OutputConfig &set(std::initializer_list Flags, + bool Value = true) { + for (auto Flag : Flags) + set(Flag, Value); + return *this; + } + + /// Set \c Flag to \c false. + constexpr OutputConfig &reset(OutputConfigFlag Flag) { + return set(Flag, false); + } + + /// Set \c Flags to \c false. + constexpr OutputConfig &reset(std::initializer_list Flags) { + return set(Flags, false); + } + + /// Nothing is set. + constexpr OutputConfig() = default; + constexpr OutputConfig(NoneType) {} + + /// Set exactly the flags listed, leaving others turned off. + constexpr OutputConfig(std::initializer_list OnFlags) { + set(OnFlags); + } + +private: + BitsetType Bits = 0; +}; + +class OutputBackend; + +/// Opaque description of a compiler output that has been created. +class OutputFile { + virtual void anchor(); + +public: + /// Internal utility for holding completed content for an output. + class ContentBuffer; + + /// Close an output, finalizing its content and sending it to the configured + /// \a OutputBackend. + /// + /// This destructs the stream if it still owns it and writes the output to + /// its final destination(s). E.g., for \a OnDiskOutputBackend, any temporary + /// files will get moved into place, or for \a InMemoryOutputBackend, the \a + /// MemoryBuffer will be created and stored in the \a InMemoryFileSystem. + /// + /// \pre If \a takeOS() has been called, the stream should have been + /// destructed before calling this function. + /// \pre \a isOpen(); i.e., neither \a erase() nor \a close() has been + /// called yet. + Error close(); + + /// Check if \a erase() or \a close() has already been called. + bool isOpen() const { return IsOpen; } + + /// Get a pointer to the output stream, if it hasn't been taken. + raw_pwrite_stream *getOS() { + initializeStreamOnce(); + return OS.get(); + } + + /// Take the output stream. This stream should be destructed before calling + /// \a close(). + /// + /// \post \a getOS() returns \c nullptr. + std::unique_ptr takeOS() { + initializeStreamOnce(); + return std::move(OS); + } + + StringRef getPath() const { return Path; } + +protected: + /// Return a content buffer stream and ask \a close() to call \a + /// storeContentBuffer(). + std::unique_ptr createStreamForContentBuffer(); + + /// Override to return a custom stream object and have \a close() call \a + /// storeStreamedContent(). The default implementation calls \a + /// initializeForContentBuffer() and forwards to \a + /// createStreamForContentBuffer(). + virtual std::unique_ptr takeOSImpl() { + return createStreamForContentBuffer(); + } + + virtual Error storeStreamedContent() { + llvm_unreachable("override this if initializeForStreamedContent() returns " + "a stream without calling createStreamForContentBuffer()"); + } + + virtual Error storeContentBuffer(ContentBuffer &Content) = 0; + + /// Allow subclasses to call \a OutputFile::storeContentBuffer() on other + /// files. + static Error storeContentBufferIn(ContentBuffer &Content, OutputFile &File) { + return File.storeContentBuffer(Content); + } + + /// Check if this is /dev/null. + virtual bool isNull() const { return false; } + + /// Allow subclasses to call \a OutputFile::isNull() on other files. + static bool isNull(const OutputFile &File) { return File.isNull(); } + + /// Allow subclasses to destroy the stream (if it's still owned), in case the + /// destructor has side effects. + void destroyOS() { OS = nullptr; } + +private: + void initializeStreamOnce() { + if (IsStreamInitialized) + return; + OS = takeOSImpl(); + IsStreamInitialized = true; + } + +public: + /// Erases the output. + virtual ~OutputFile(); + + OutputFile(OutputFile &&O) = delete; + OutputFile &operator=(OutputFile &&O) = delete; + +protected: + explicit OutputFile(StringRef Path) : Path(Path.str()) {} + +private: + /// Output path. + std::string Path; + + /// Tracks whether the output is still open, before one of \a erase() or \a + /// close() is called. + bool IsOpen = true; + + /// Track whether this has been initialized. + bool IsStreamInitialized = false; + + /// Content buffer if the output destination requests one. Behind a pointer + /// in case the output is moved, since the vector needs a stable address. + std::unique_ptr> Bytes; + + // Destroyed ContentBuffer since it can reference it. + std::unique_ptr OS; +}; + +class OutputDirectory; + +/// Backend interface for \a OutputBackend. Its job is to generate \a +/// OutputFileImpl given an \a OutputPath and \c OutputConfig. +class OutputBackend : public RefCountedBase { + virtual void anchor(); + +public: + Expected> createFile(const Twine &Path, + OutputConfig Config = None); + + /// Get the \a OutputDirectory for \p Path, which is an \a OutputBackend that + /// uses \c Path as the current working directory. \p Path does not need to + /// exist. This only fails when the call to \a resolveOutputPath() fails. + Expected> + getDirectory(const Twine &Path) const; + + /// Crete the \a OutputDirectory for \p Path, which is an \a OutputBackend + /// that uses \c Path as the current working directory. + /// + /// Not all backends have a concept of directories existing without files in + /// them. + Expected> + createDirectory(const Twine &Path, OutputConfig Config = None); + + /// Create a unique file. + /// + /// Not all backends have access to an input filesystem. + Expected> + createUniqueFile(const Twine &Model, OutputConfig Config = None); + + Expected> + createUniqueFile(const Twine &Prefix, StringRef Suffix, + OutputConfig Config = None) { + return createUniqueFile(Prefix + "-%%%%%%%%" + Suffix, Config); + } + + Expected> + createUniqueDirectory(const Twine &Prefix, OutputConfig Config = None); + + sys::path::Style getPathStyle() const { return PathStyle; } + + /// A non-binding request to resolve \p Path. Output backends with a working + /// directory will typically make relative paths absolute here. For example, + /// see \a OutputDirectoryAdaptorBase::resolveOutputPathImpl(). + /// + /// Called on all paths before forwarding to \a createFileImpl(), + /// \a createUniqueFileImpl(), \a createUniqueDirectoryImpl(), or + /// \a createDirectoryImpl(). + virtual Expected + resolveOutputPath(const Twine &Path, + SmallVectorImpl &PathStorage) const; + +protected: + /// Create an \a OutputDirectory without checking if the directory exists. + virtual IntrusiveRefCntPtr + getDirectoryImpl(StringRef ResolvedPath) const; + + /// Create an output. Returning \c nullptr will cause the output to be + /// ignored. Implementations should not call \a initialize(); that's done by + /// \a createFile(). + /// + /// \return A valid \a OutputFile, \c nullptr, or an \a Error. + virtual Expected> + createFileImpl(StringRef ResolvedPath, OutputConfig Config) = 0; + + virtual Expected> + createUniqueFileImpl(StringRef ResolvedModel, OutputConfig Config) = 0; + + /// Default implementation assumes that directory creation is a no-op, and + /// returns a just-constructed \a OutputDirectory with \p ResolvedPath. + virtual Expected> + createDirectoryImpl(StringRef ResolvedPath, OutputConfig) { + return getDirectory(ResolvedPath); + } + + virtual Expected> + createUniqueDirectoryImpl(StringRef ResolvedPrefix, OutputConfig Config) = 0; + +public: + OutputBackend() = delete; + explicit OutputBackend(sys::path::Style PathStyle) : PathStyle(PathStyle) {} + + virtual ~OutputBackend() = default; + +private: + sys::path::Style PathStyle; +}; + +/// Helper class for creating proxies of another backend. +class ProxyOutputBackend : public OutputBackend { +protected: + Expected + resolveOutputPath(const Twine &Path, + SmallVectorImpl &PathStorage) const override { + return UnderlyingBackend->resolveOutputPath(Path, PathStorage); + } + + Expected> + createFileImpl(StringRef OutputPath, OutputConfig Config) override { + return UnderlyingBackend->createFile(OutputPath, Config); + } + + Expected> + createDirectoryImpl(StringRef ResolvedPath, OutputConfig Config) override { + return UnderlyingBackend->createDirectory(ResolvedPath, Config); + } + + Expected> + createUniqueFileImpl(StringRef ResolvedModel, OutputConfig Config) override { + return UnderlyingBackend->createUniqueFile(ResolvedModel, Config); + } + + Expected> + createUniqueDirectoryImpl(StringRef ResolvedPrefix, + OutputConfig Config) override { + return UnderlyingBackend->createUniqueDirectory(ResolvedPrefix, Config); + } + + OutputBackend &getUnderlyingBackend() { return *UnderlyingBackend; } + +public: + ProxyOutputBackend(IntrusiveRefCntPtr UnderlyingBackend) + : OutputBackend(UnderlyingBackend->getPathStyle()), + UnderlyingBackend(std::move(UnderlyingBackend)) { + assert(this->UnderlyingBackend && "Expected valid underlying backend"); + } + +private: + IntrusiveRefCntPtr UnderlyingBackend; +}; + +class OutputDirectoryAdaptorBase { +protected: + /// If \p Path is relative and has a compatible root name with \a Directory, + /// prepend \a Directory to make it absolute. Otherwise, do nothing. + Expected resolveOutputPathImpl(const Twine &Path, + SmallVectorImpl &PathStorage, + sys::path::Style PathStyle) const; + +public: + OutputDirectoryAdaptorBase(const Twine &Directory) + : Directory(Directory.str()) {} + + StringRef getPath() const { return Directory; } + +private: + std::string Directory; +}; + +/// An adaptor to add a fixed working directory to an output backend. +template +class OutputDirectoryAdaptor : public BaseBackendT, OutputDirectoryAdaptorBase { +protected: + using OutputDirectoryAdaptorType = OutputDirectoryAdaptor; + +public: + Expected + resolveOutputPath(const Twine &Path, + SmallVectorImpl &PathStorage) const override { + return OutputDirectoryAdaptorBase::resolveOutputPathImpl( + Path, PathStorage, BaseBackendT::getPathStyle()); + } + + using OutputDirectoryAdaptorBase::getPath; + + template + OutputDirectoryAdaptor(const Twine &DirectoryPath, ArgsT &&... Args) + : BaseBackendT(std::forward(Args)...), + OutputDirectoryAdaptorBase(DirectoryPath) {} +}; + +/// An output backend proxy with a fixed workign directory. +class OutputDirectory : public OutputDirectoryAdaptor { + virtual void anchor(); + +public: + /// Construct an output directory for a given backend. Does not check or + /// modify \p DirectoryPath, just assumes it's valid and absolute-enough. + /// + /// Most users will want \a OutputBackend::getDirectory(), which first + /// resolves the working directory, or \a OutputBackend::createDirectory(), + /// which also explicit requests creating it. + OutputDirectory(IntrusiveRefCntPtr UnderlyingBackend, + const Twine &DirectoryPath) + : OutputDirectoryAdaptorType(DirectoryPath, + std::move(UnderlyingBackend)) {} +}; + +/// A helper class for creating unique files and directories. +class StableUniqueEntityHelper { +public: + StringRef getNext(StringRef Model, SmallVectorImpl &PathStorage); + + Expected> + createStableUniqueFile(OutputBackend &Backend, StringRef Model, + OutputConfig Config); + + Expected> + createStableUniqueDirectory(OutputBackend &Backend, StringRef Model, + OutputConfig Config); + +private: + StringMap Models; +}; + +/// An adaptor to create a default implementation of \a +/// OutputBackend::createUniqueFileImpl() and \a +/// OutputBackend::createUniqueDirectoryImpl(). +template +class StableUniqueEntityAdaptor : public BaseBackendT, + StableUniqueEntityHelper { +protected: + using StableUniqueEntityAdaptorType = StableUniqueEntityAdaptor; + + Expected> + createUniqueFileImpl(StringRef Model, OutputConfig Config) override { + return this->createStableUniqueFile(*this, Model, Config); + } + + Expected> + createUniqueDirectoryImpl(StringRef Model, OutputConfig Config) override { + return this->createStableUniqueDirectory(*this, Model, Config); + } + +public: + template + StableUniqueEntityAdaptor(ArgsT &&... Args) + : BaseBackendT(std::forward(Args)...) {} +}; + +/// Interface for managing the destination of an \a OutputFile. Most users only +/// need to deal with \a OutputFile. +/// +/// \a OutputFileImpl's lifetime is expected to follow one of the following +/// "good" paths after construction: +/// - \a takeStream() is called and returns a valid stream. The caller writes +/// content to the stream, destructs it, and then calls \a +/// storeStreamedContent() is to store the content. +/// - \a takeStream() is called and returns \c nullptr. The caller collects the +/// content and calls \a storeContent() to store it. +/// - \a storeContent() is called without first calling \a takeStream(), as the +/// caller wants to pass in the completed content as a whole instead of +/// streaming. +/// +/// If the destination is destructed before calling \a storeStreamedContent() +/// and \a storeContent(), this output will be cancelled and temporaries +/// cleaned up. +/// +/// \a storeContent() is designed to allow output destinations to be chained, +/// passing content between them. \a ContentBuffer helps to manage the lifetime +/// of the content, copying data and constructing memory buffers only as +/// needed. +class OutputFile::ContentBuffer { +public: + StringRef getBytes() const { return Bytes; } + + /// Whether the content is currently owned by this utility. + bool ownsContent() const { return Vector || OwnedBuffer; } + + /// Returns a valid \a MemoryBuffer. If \a ownsContent() is false, + /// this buffer is a reference constructed on the fly using \a + /// MemoryBuffer::getMemBuffer(). + /// + /// \post \b ownsContent() is false. + std::unique_ptr takeBuffer(StringRef Identifier); + + /// Returns a valid \a MemoryBuffer that owns its bytes, or \c + /// nullptr if \a ownsContent() is \c false. + /// + /// \post \b ownsContent() is false. + std::unique_ptr takeOwnedBufferOrNull(StringRef Identifier); + + /// Returns a valid \a MemoryBuffer that owns its bytes, or calls + /// MemoryBuffer::getMemBufferCopy() if \a ownsContent() is \c false. + /// + /// \post \b ownsContent() is false. + std::unique_ptr takeOwnedBufferOrCopy(StringRef Identifier); + + /// Construct a reference to content owned by someone else. + /// + /// \post \b ownsContent() is false. + ContentBuffer(StringRef Bytes) : Bytes(Bytes) { + assert(!this->Bytes.end()[0] && "Requires null terminator"); + } + + /// Construct a buffer named \p Identifier from \p Vector. This buffer is + /// created lazily on the first call to \a takeBuffer() and friends using + /// \a SmallVectorMemoryBuffer. + /// + /// \post \b ownsContent() is true, unless \c Vector.empty(). + /// \post \a getBytes() returns a range with the memory from \p Vector, + /// unless \p Vector was empty or in small mode. + ContentBuffer(SmallVectorImpl &&Vector) : Vector(std::move(Vector)) { + finishConstructingFromVector(); + assert(!Bytes.end()[0] && "Requires null terminator"); + } + + /// Store \p Buffer as the content, returned by the first call to \a + /// takeBuffer() and friends. + /// + /// \pre \p Buffer is null-terminated. + /// \post \b ownsContent() is true. + ContentBuffer(std::unique_ptr Buffer) + : Bytes(Buffer->getBuffer()), OwnedBuffer(std::move(Buffer)) { + assert(!Bytes.end()[0] && "Requires null terminator"); + } + + ContentBuffer(ContentBuffer &&) = default; + ContentBuffer &operator=(ContentBuffer &&) = default; + +private: + ContentBuffer() = delete; + ContentBuffer(const ContentBuffer &&) = delete; + ContentBuffer &operator=(const ContentBuffer &&) = delete; + + /// Main body of constructor when initializing \a Vector. + void finishConstructingFromVector(); + + /// Reference to the bytes. Must be null-terminated. + StringRef Bytes; + + /// Owned content stored in a vector. + Optional> Vector; + + /// Owned content stored in a memory buffer. + std::unique_ptr OwnedBuffer; +}; + +/// Base class for OutputBackend errors. +class OutputError : public ErrorInfo { + void anchor() override; + +public: + StringRef getOutputPath() const { return OutputPath; } + std::error_code getErrorCode() const { return EC; } + + std::error_code convertToErrorCode() const override { return getErrorCode(); } + + // Used by ErrorInfo::classID. + static char ID; + +protected: + OutputError(StringRef OutputPath, std::error_code EC) + : OutputPath(OutputPath), EC(EC) { + assert(EC && "Cannot create OutputError from success EC"); + } + +private: + std::string OutputPath; + std::error_code EC; +}; + +/// The output already exists in filesystem and it cannot be overwritten. +class CannotOverwriteExistingOutputError final + : public ErrorInfo { + friend Error createCannotOverwriteExistingOutputError(StringRef OutputPath); + + void anchor() override; + +public: + void log(raw_ostream &OS) const override { + OS << "'" << getOutputPath() << "' already exists in-memory"; + } + + // Used by ErrorInfo::classID. + static char ID; + +private: + CannotOverwriteExistingOutputError(StringRef OutputPath) + : ErrorInfo( + OutputPath, std::make_error_code(std::errc::file_exists)) {} +}; + +inline Error createCannotOverwriteExistingOutputError(StringRef OutputPath) { + return Error(std::unique_ptr( + new CannotOverwriteExistingOutputError(OutputPath))); +} + +/// A temporary file could not be renamed to the final output on disk. +class OnDiskOutputRenameTempError final + : public ErrorInfo { + friend Error createOnDiskOutputRenameTempError(StringRef TempPath, + StringRef OutputPath, + std::error_code EC); + void anchor() override; + +public: + void log(raw_ostream &OS) const override { + assert(getErrorCode()); + OS << "'" << TempPath << "' could not be renamed to "; + OS << "'" << getOutputPath() << "': "; + OS << getErrorCode().message(); + } + + StringRef getTempPath() const { return TempPath; } + + // Used by ErrorInfo::classID. + static char ID; + +private: + OnDiskOutputRenameTempError(StringRef TempPath, StringRef OutputPath, + std::error_code EC) + : ErrorInfo(OutputPath, EC), + TempPath(TempPath) { + assert(EC && "Cannot create OnDiskOutputRenameTempError from success EC"); + } + + std::string TempPath; +}; + +inline Error createOnDiskOutputRenameTempError(StringRef TempPath, + StringRef OutputPath, + std::error_code EC) { + return Error(std::unique_ptr( + new OnDiskOutputRenameTempError(TempPath, OutputPath, EC))); +} + +class OnDiskOutputBackend : public OutputBackend { + void anchor() override; + +protected: + Expected> + createFileImpl(StringRef ResolvedPath, OutputConfig Config) override; + + Expected> + createDirectoryImpl(StringRef ResolvedPath, OutputConfig Config) override; + + Expected> + createUniqueFileImpl(StringRef ResolvedModel, OutputConfig Config) override; + + Expected> + createUniqueDirectoryImpl(StringRef ResolvedPrefix, + OutputConfig Config) override; + +public: + /// Big enough that mmap won't use up too much address space. + static constexpr unsigned MinimumSizeToReturnWriteThroughBuffer = 4 * 4096; + + /// On disk output settings. + struct OutputSettings { + /// Register output files to be deleted if a signal is received. Disabled + /// for outputs with \a OutputConfigFlag::NoCrashCleanup. + bool RemoveOnSignal = true; + + /// Use stable entity names for \a createUniqueFileImpl() and \a + /// createUniqueDirectoryImpl(). + bool StableUniqueEntities = false; + struct { + bool Use = false; + + // If a write-through buffer is used, try to forward it. Disabled for + // outputs with \a OutputConfigFlag::Volatile. + bool Forward = true; + } WriteThrough; + }; + OutputSettings &getSettings() { return Settings; } + const OutputSettings &getSettings() const { return Settings; } + + /// Get a view of this backend that has stable unique entity names. + IntrusiveRefCntPtr createStableUniqueEntityView() { + return makeIntrusiveRefCnt>( + this); + } + + OnDiskOutputBackend() : OutputBackend(sys::path::Style::native) {} + +private: + OutputSettings Settings; +}; + +/// Create a backend that ignores all output. +IntrusiveRefCntPtr +makeNullOutputBackend(sys::path::Style PathStyle = sys::path::Style::native); + +/// Create a backend that mirrors content between \a Backend1 and \a +/// Backend2. +/// +/// Both backends are asked to create destinations for each output. \a Backend1 +/// receives the content before \a Backend2. +IntrusiveRefCntPtr +makeMirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2); + +/// A backend for storing outputs in an instance of \a InMemoryFileSystem. +class InMemoryOutputBackend : public StableUniqueEntityAdaptor<> { + void anchor() override; + +protected: + Expected> + createFileImpl(StringRef OutputPath, OutputConfig Config) override; + + /// Get the installed in-memory filesystem, if any. + InMemoryFileSystem &getInMemoryFS() const { return *FS; } + +public: + struct OutputSettings { + bool FailIfExists = false; + bool CopyUnownedBuffers = false; + }; + OutputSettings &getSettings() { return Settings; } + const OutputSettings &getSettings() const { return Settings; } + + InMemoryOutputBackend(IntrusiveRefCntPtr FS, + sys::path::Style PathStyle = sys::path::Style::native) + : StableUniqueEntityAdaptorType(PathStyle), FS(std::move(FS)) {} + +private: + OutputSettings Settings; + + /// In-memory filesystem for writing outputs to. + IntrusiveRefCntPtr FS; +}; + +/// Create an adaptor backend that filters the outputs that written to the +/// underlying backend. Outputs where \p Filter returns \c false will be +/// ignored. +IntrusiveRefCntPtr makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + unique_function Filter); + +} // namespace vfs +} // namespace llvm + +#endif // LLVM_SUPPORT_OUTPUTBACKEND_H Index: llvm/lib/Support/CMakeLists.txt =================================================================== --- llvm/lib/Support/CMakeLists.txt +++ llvm/lib/Support/CMakeLists.txt @@ -152,6 +152,7 @@ NativeFormatting.cpp OptimizedStructLayout.cpp Optional.cpp + OutputBackend.cpp Parallel.cpp PluginLoader.cpp PrettyStackTrace.cpp Index: llvm/lib/Support/OutputBackend.cpp =================================================================== --- /dev/null +++ llvm/lib/Support/OutputBackend.cpp @@ -0,0 +1,986 @@ +//===- OutputBackend.cpp - Manage compiler outputs ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the OutputBackend interface. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/OutputBackend.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::vfs; + +void OutputFile::anchor() {} +void OutputBackend::anchor() {} +void OutputDirectory::anchor() {} +void OnDiskOutputBackend::anchor() {} +void InMemoryOutputBackend::anchor() {} + +void OutputError::anchor() {} +char OutputError::ID = 0; + +void CannotOverwriteExistingOutputError::anchor() {} +char CannotOverwriteExistingOutputError::ID = 0; + +void OnDiskOutputRenameTempError::anchor() {} +char OnDiskOutputRenameTempError::ID = 0; + +OutputFile::~OutputFile() = default; + +std::unique_ptr OutputFile::createStreamForContentBuffer() { + Bytes = std::make_unique>(); + return std::make_unique(*Bytes); +} + +Error OutputFile::close() { + assert(isOpen() && "Output already closed"); + + // Destruct the stream if we still own it. + OS = nullptr; + IsOpen = false; + + // If there's no content buffer this is using a stream. + if (!Bytes) + return storeStreamedContent(); + + // If there's a content buffer, send it along. + OutputFile::ContentBuffer Buffer(std::move(*Bytes)); + return storeContentBuffer(Buffer); +} + +void OutputFile::ContentBuffer::finishConstructingFromVector() { + // If Vector is empty, we can't guarantee SmallVectorMemoryBuffer will keep + // pointer identity. But since there's no content, it's all moot; just + // construct a reference. + if (Vector->empty()) { + // Drop the vector and use a null-terminated string for the bytes. + Vector = None; + Bytes = ""; + return; + } + + // Null-terminate and set the bytes. + Vector->push_back(0); + Vector->pop_back(); + Bytes = StringRef(Vector->begin(), Vector->size()); +} + +namespace { +class RenamedMemoryBuffer : public llvm::MemoryBuffer { +public: + StringRef getBufferIdentifier() const override { return Identifier; } + BufferKind getBufferKind() const override { + return UnderlyingMB->getBufferKind(); + } + + RenamedMemoryBuffer(std::unique_ptr UnderlyingMB, + StringRef Identifier) + : Identifier(Identifier.str()), UnderlyingMB(std::move(UnderlyingMB)) { + init(this->UnderlyingMB->getBufferStart(), + this->UnderlyingMB->getBufferEnd(), + /*RequiresNullTerminator=*/true); + } + +private: + std::string Identifier; + std::unique_ptr UnderlyingMB; +}; +} // namespace + +std::unique_ptr +OutputFile::ContentBuffer::takeOwnedBufferOrNull(StringRef Identifier) { + if (OwnedBuffer) { + // Ensure the identifier is correct. + if (OwnedBuffer->getBufferIdentifier() == Identifier) + return std::move(OwnedBuffer); + return std::make_unique(std::move(OwnedBuffer), + Identifier); + } + + if (!Vector) + return nullptr; + + bool IsEmpty = Vector->empty(); + (void)IsEmpty; + auto VectorBuffer = + std::make_unique(std::move(*Vector), Identifier); + Vector = None; + assert(Bytes.begin() == VectorBuffer->getBufferStart()); + assert(Bytes.end() == VectorBuffer->getBufferEnd()); + return VectorBuffer; +} + +std::unique_ptr +OutputFile::ContentBuffer::takeBuffer(StringRef Identifier) { + if (std::unique_ptr B = takeOwnedBufferOrNull(Identifier)) + return B; + return MemoryBuffer::getMemBuffer(Bytes, Identifier); +} + +std::unique_ptr +OutputFile::ContentBuffer::takeOwnedBufferOrCopy(StringRef Identifier) { + if (std::unique_ptr B = takeOwnedBufferOrNull(Identifier)) + return B; + return MemoryBuffer::getMemBufferCopy(Bytes, Identifier); +} + +static std::unique_ptr makeNullOutput(StringRef Path) { + class NullOutput : public OutputFile { + std::unique_ptr takeOSImpl() override { + return std::make_unique(); + } + Error storeStreamedContent() override { return Error::success(); } + Error storeContentBuffer(ContentBuffer &) override { + return Error::success(); + } + bool isNull() const override { return true; } + + public: + NullOutput(StringRef Path) : OutputFile(Path) {} + }; + return std::make_unique(Path); +} + +Expected +OutputBackend::resolveOutputPath(const Twine &Path, + SmallVectorImpl &PathStorage) const { + return Path.toStringRef(PathStorage); +} + +IntrusiveRefCntPtr +OutputBackend::getDirectoryImpl(StringRef ResolvedPath) const { + return makeIntrusiveRefCnt(const_cast(this), + ResolvedPath); +} + +Expected> +OutputBackend::getDirectory(const Twine &Path) const { + SmallString<128> Storage; + if (Expected E = resolveOutputPath(Path, Storage)) + return getDirectoryImpl(*E); + else + return E.takeError(); +} + +Expected> +OutputBackend::createFile(const Twine &Path, OutputConfig Config) { + SmallString<128> Storage; + if (Expected E = resolveOutputPath(Path, Storage)) + return createFileImpl(*E, Config); + else + return E.takeError(); +} + +Expected> +OutputBackend::createDirectory(const Twine &Path, OutputConfig Config) { + SmallString<128> Storage; + if (Expected E = resolveOutputPath(Path, Storage)) + return createDirectoryImpl(*E, Config); + else + return E.takeError(); +} + +Expected> +OutputBackend::createUniqueFile(const Twine &Model, OutputConfig Config) { + SmallString<128> Storage; + if (Expected E = resolveOutputPath(Model, Storage)) + return createUniqueFileImpl(*E, Config); + else + return E.takeError(); +} + +Expected> +OutputBackend::createUniqueDirectory(const Twine &Model, OutputConfig Config) { + SmallString<128> Storage; + if (Expected E = resolveOutputPath(Model, Storage)) + return createUniqueDirectoryImpl(*E, Config); + else + return E.takeError(); +} + +Expected OutputDirectoryAdaptorBase::resolveOutputPathImpl( + const Twine &PathTwine, SmallVectorImpl &PathStorage, + sys::path::Style PathStyle) const { + StringRef Path = PathTwine.toStringRef(PathStorage); + if (sys::path::is_absolute(Path, PathStyle)) + return Path; + + // Return Path as-is if it has a root-name and it doesn't match + // Directory's root name, indicating that Directory and Path are on different + // drives. + // + // Note: This is intentionally asymmetric: a Directory without a root name + // is not prepended if Path has a root name. + StringRef PathRootName = sys::path::root_name(Path, PathStyle); + if (!PathRootName.empty() && + PathRootName != sys::path::root_name(Directory, PathStyle)) + return Path; + + // Root names match. Make the path absolute. + SmallString<128> ResolvedPath = StringRef(Directory); + sys::path::append(ResolvedPath, PathStyle, Path); + ResolvedPath.swap(PathStorage); + return StringRef(PathStorage.begin(), PathStorage.size()); +} + +StringRef +StableUniqueEntityHelper::getNext(StringRef Model, + SmallVectorImpl &PathStorage) { + unsigned N = Models[Model]++; + + // Replace '%' with chars representing N in hex. + PathStorage.assign(Model.begin(), Model.end()); + for (auto I = PathStorage.rbegin(), E = PathStorage.rend(); I != E; ++I) { + if (*I != '%') + continue; + *I = "0123456789abcdef"[N & 15]; + N >>= 4; + } + + return StringRef(PathStorage.begin(), PathStorage.size()); +} + +template +static Expected createStableUniqueEntity( + StringRef Model, OutputConfig Config, + llvm::function_ref &)> GetNext, + llvm::function_ref(StringRef, OutputConfig)> CreateEntity) { + // Turn off overwriting and try models one at a time. + Config.set(OutputConfigFlag::NoOverwrite); + int NumTries = 0; + SmallString<128> PathStorage; + for (;;) { + PathStorage.clear(); + StringRef Path = GetNext(Model, PathStorage); + auto ExpectedEntity = CreateEntity(Path, Config); + if (ExpectedEntity) + return std::move(*ExpectedEntity); + + Error E = ExpectedEntity.takeError(); + if (++NumTries >= 128) + return std::move(E); + + // FIXME: Return E if it can't indicate a failure to clobber an existing + // entity; otherwise consume it and contintue. + // + // For OnDisk, we'd get: + // - errc::file_exists in typical cases + // - errc::permission_denied on Windows when marked for deletion + // + // But maybe we should rely on the backend being more clear. + } +} + +Expected> +StableUniqueEntityHelper::createStableUniqueFile(OutputBackend &Backend, + StringRef Model, + OutputConfig Config) { + return createStableUniqueEntity>( + Model, Config, + [this](StringRef Model, SmallVectorImpl &PathStorage) { + return getNext(Model, PathStorage); + }, + [&Backend](StringRef Path, OutputConfig Config) { + return Backend.createFile(Path, Config); + }); +} + +Expected> +StableUniqueEntityHelper::createStableUniqueDirectory(OutputBackend &Backend, + StringRef Model, + OutputConfig Config) { + return createStableUniqueEntity>( + Model, Config, + [this](StringRef Model, SmallVectorImpl &PathStorage) { + return getNext(Model, PathStorage); + }, + [&Backend](StringRef Path, OutputConfig Config) { + return Backend.createDirectory(Path, Config); + }); +} + +namespace { +class NullOutputBackend : public StableUniqueEntityAdaptor<> { +public: + Expected> createFileImpl(StringRef Path, + OutputConfig) override { + return makeNullOutput(Path); + } + + explicit NullOutputBackend(sys::path::Style PathStyle) + : StableUniqueEntityAdaptorType(PathStyle) {} +}; +} // anonymous namespace + +IntrusiveRefCntPtr +llvm::vfs::makeNullOutputBackend(sys::path::Style PathStyle) { + return makeIntrusiveRefCnt(PathStyle); +} + +namespace { +class OnDiskOutputFile : public OutputFile { +public: + /// Open a file on-disk for writing to \a OutputPath. + /// + /// This calls \a initializeFD() to initialize \a FD (see that + /// documentation for more detail) and \a UseWriteThroughBuffer. Unless \a + /// UseWriteThroughBuffer was determined to be \c true, this function will + /// then initialize \a OS with a valid stream. + Error initializeFile(); + + /// Erases the file if it hasn't be closed. If \a TempFile is set, erases it; + /// otherwise, erases \a OutputPath. + ~OnDiskOutputFile() override; + + /// Take an open output stream. + std::unique_ptr takeOSImpl() override; + + Error storeStreamedContent() override { return closeFile(nullptr); } + + Error storeContentBuffer(ContentBuffer &Content) override { + return closeFile(&Content); + } + +private: + using CheckingContentConsumerType = + function_ref)>; + + /// Close a file after successfully collecting the output. + /// + /// The output content must have already been written somewhere, but + /// exactly where depends on the configuration. + /// + /// - If \a WriteThroughMemoryBuffer is \c true, the output will be in \a + /// Content and \a FD has an open file descriptor. In that case, the file + /// will be completed by opening \a WriteThroughMemoryBuffer and + /// calling \a std::memcpy(). + /// - Else if the output is in \a Content, it will be written to \a OS, + /// which should be an already-open file stream, and the content will be + /// written there and the stream destroyed to flush it. + /// - Else the content should already be in the file. + /// + /// Once the content is on-disk, if \a TempPath is set, this calls \a + /// sys::rename() to move the file to \a OutputPath. + /// + /// Returns a file-backed MemoryBuffer when the file was written using a + /// write-through memory buffer, unless it won't be null-terminated or it's + /// too small. Otherwise, when there's no error, returns \c nullptr. + Error closeFile(ContentBuffer *MaybeContent); + + Expected> + openMappedFile(size_t Size); + + std::unique_ptr convertMappedFileToBuffer( + std::unique_ptr Mapping); + + /// Attempt to open a temporary file for \a OutputPath. + /// + /// This tries to open a uniquely-named temporary file for \a OutputPath, + /// possibly also creating any missing directories if \a + /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a + /// Config. + /// + /// \post FD and \a TempPath are initialized if this is successful. + std::error_code tryToCreateTemporary(); + + /// Open a file on-disk for writing to OutputPath. + /// + /// This opens a file for writing and assigns the file descriptor to \c FD. + /// Exactly how the file is opened depends on the \a OnDiskOutputConfig + /// settings in \p Config and (if the file exists) the type of file. + /// + /// - If \a getPath() is \c "-" (indicating stdin), this function returns + /// \a Error::success() but has no effect. + /// - Unless \a OutputConfigFlag::NoAtomicWrite, a temporary file is opened + /// first, to be renamed to \a getPath() when \a closeFile() is called. + /// This is disabled if \a getPath() exists and \a + /// sys::fs::is_regular_file() returns \c false (such as a named pipe). If + /// \a tryToCreateTemporary() fails, this falls back to no-temp-file mode. + /// (Even if using a temporary file, \a getPath() is checked for write + /// permission.) + /// - \a OnDiskOutputConfig::RemoveFileOnSignal installs a signal handler + /// to remove the opened file. + /// + /// This function also validates \c Settings.WriteThrough.Use based on the + /// output file type. + /// + /// \post FD is set unless \a OutputPath is \c "-" or on error. + /// \post TempPath is set if a temporary file was opened successfully. + Error initializeFD(); + + void initializeExistingFD(); + +public: + OnDiskOutputFile(StringRef Path, OutputConfig Config, + OnDiskOutputBackend::OutputSettings Settings) + : OutputFile(Path), Settings(Settings), Config(Config) {} + + /// Construct from an already-open file descriptor. + OnDiskOutputFile(int FD, StringRef Path, OutputConfig Config, + OnDiskOutputBackend::OutputSettings Settings) + : OutputFile(Path), Settings(Settings), Config(Config), FD(FD) { + initializeExistingFD(); + } + +private: + OnDiskOutputBackend::OutputSettings Settings; + OutputConfig Config; + std::unique_ptr OS; + Optional TempPath; + Optional FD; +}; +} // anonymous namespace + +std::error_code OnDiskOutputFile::tryToCreateTemporary() { + // Create a temporary file. + // Insert -%%%%%%%% before the extension (if any), and because some tools + // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build + // artifacts, also append .tmp. + StringRef OutputExtension = sys::path::extension(getPath()); + SmallString<128> TempPath = + StringRef(getPath()).drop_back(OutputExtension.size()); + TempPath += "-%%%%%%%%"; + TempPath += OutputExtension; + TempPath += ".tmp"; + + auto tryToCreateImpl = [&]() { + int NewFD; + if (std::error_code EC = + sys::fs::createUniqueFile(TempPath, NewFD, TempPath)) + return EC; + if (!Config.test(OutputConfigFlag::NoCrashCleanup)) + sys::RemoveFileOnSignal(TempPath); + this->TempPath = TempPath.str().str(); + FD = NewFD; + return std::error_code(); + }; + + // Try to create the temporary. + std::error_code EC = tryToCreateImpl(); + if (!EC) + return std::error_code(); + + if (EC != std::errc::no_such_file_or_directory || + Config.test(OutputConfigFlag::NoImplyCreateDirectories)) + return EC; + + // Create parent directories and try again. + StringRef ParentPath = sys::path::parent_path(getPath()); + if ((EC = sys::fs::create_directories(ParentPath))) + return EC; + return tryToCreateImpl(); +} + +Error OnDiskOutputFile::initializeFD() { + // Disable temporary file for stdout (and return early since we won't use a + // file descriptor directly). + if (getPath() == "-") { + Settings.WriteThrough.Use = false; + return Error::success(); + } + + // Function to check and update whether to use a write-through buffer. + bool CheckedStatusForWriteThrough = false; + auto checkStatusForWriteThrough = [&](const sys::fs::file_status &Status) { + if (CheckedStatusForWriteThrough) + return; + sys::fs::file_type Type = Status.type(); + if (Type != sys::fs::file_type::regular_file && + Type != sys::fs::file_type::block_file) + Settings.WriteThrough.Use = false; + CheckedStatusForWriteThrough = true; + }; + + // Disable temporary file for other non-regular files, and if we get a status + // object, also check if we can write and disable write-through buffers if + // appropriate. + if (!Config.test(OutputConfigFlag::NoAtomicWrite)) { + sys::fs::file_status Status; + sys::fs::status(getPath(), Status); + if (sys::fs::exists(Status)) { + if (!sys::fs::is_regular_file(Status)) + Config.set(OutputConfigFlag::NoAtomicWrite); + + // Fail now if we can't write to the final destination. + if (!sys::fs::can_write(getPath())) + return errorCodeToError( + std::make_error_code(std::errc::operation_not_permitted)); + + checkStatusForWriteThrough(Status); + } + } + + // If (still) using a temporary file, try to create it (and return success if + // that works). + if (!Config.test(OutputConfigFlag::NoAtomicWrite)) + if (!tryToCreateTemporary()) + return Error::success(); + + // Not using a temporary file. Open the final output file. + int NewFD; + if (auto EC = sys::fs::openFileForWrite( + getPath(), NewFD, sys::fs::CD_CreateAlways, + Config.test(OutputConfigFlag::Text) ? sys::fs::OF_Text + : sys::fs::OF_None)) + return errorCodeToError(EC); + FD = NewFD; + + // Check the status with the open FD to see whether a write-through buffer + // makes sense. + if (Settings.WriteThrough.Use && !CheckedStatusForWriteThrough) { + sys::fs::file_status Status; + sys::fs::status(NewFD, Status); + checkStatusForWriteThrough(Status); + } + + if (!Config.test(OutputConfigFlag::NoCrashCleanup)) + sys::RemoveFileOnSignal(getPath()); + return Error::success(); +} + +void OnDiskOutputFile::initializeExistingFD() { + this->Config.set(OutputConfigFlag::NoAtomicWrite); + if (!Config.test(OutputConfigFlag::NoCrashCleanup)) + sys::RemoveFileOnSignal(getPath()); +} + +Error OnDiskOutputFile::initializeFile() { + if (!FD) + if (Error E = initializeFD()) + return E; + + assert(FD || getPath() == "-"); + if (Settings.WriteThrough.Use) + return Error::success(); + + // Open the raw_fd_ostream right away to free the file descriptor. + std::error_code EC; + OS = getPath() == "-" + ? std::make_unique(getPath(), EC) + : std::make_unique(*FD, /*shouldClose=*/true); + assert(!EC && "Unexpected error opening stdin"); + + return Error::success(); +} + +std::unique_ptr OnDiskOutputFile::takeOSImpl() { + if (Settings.WriteThrough.Use) + return createStreamForContentBuffer(); + + assert(OS && "Expected file to be initialized"); + + // Check whether we can get away with returning the stream directly. Note + // that we don't need seeking for Text files. + if (OS->supportsSeeking() || Config.test(OutputConfigFlag::Text)) + return std::move(OS); + + // Fall back on the content buffer. + return createStreamForContentBuffer(); +} + +std::unique_ptr OnDiskOutputFile::convertMappedFileToBuffer( + std::unique_ptr Mapping) { + assert(Mapping->const_data()[Mapping->size()] == 0 && + "Expected mmap region to be null-terminated"); + + class MappedMemoryBuffer : public llvm::MemoryBuffer { + public: + StringRef getBufferIdentifier() const override { return Path; } + BufferKind getBufferKind() const override { return MemoryBuffer_MMap; } + + MappedMemoryBuffer(StringRef Path, + std::unique_ptr Mapping) + : Path(Path.str()), Mapping(std::move(Mapping)) { + const char *Data = this->Mapping->const_data(); + init(Data, Data + this->Mapping->size(), /*RequiresNullTerminator=*/true); + } + + private: + std::string Path; + std::unique_ptr Mapping; + }; + return std::make_unique(getPath(), std::move(Mapping)); +} + +Expected> +OnDiskOutputFile::openMappedFile(size_t Size) { + if (auto EC = sys::fs::resize_file_before_mapping_readwrite(*FD, Size)) + return errorCodeToError(EC); + + std::error_code EC; + auto Mapping = std::make_unique( + sys::fs::convertFDToNativeFile(*FD), + sys::fs::mapped_file_region::readwrite, Size, 0, EC); + if (EC) + return errorCodeToError(EC); + return Mapping; +} + +Error OnDiskOutputFile::closeFile(ContentBuffer *MaybeContent) { + Optional BufferedContent; + if (MaybeContent) + BufferedContent = MaybeContent->getBytes(); + if (Settings.WriteThrough.Use) { + assert(FD && "Write-through buffer needs a file descriptor"); + assert(!OS && "Expected no stream when using write-through buffer"); + assert(BufferedContent && "Expected buffered content"); + std::unique_ptr Mapping; + if (auto ExpectedMapping = openMappedFile(BufferedContent->size())) + Mapping = std::move(*ExpectedMapping); + else + return ExpectedMapping.takeError(); + + std::memcpy(Mapping->data(), BufferedContent->begin(), + BufferedContent->size()); + + // Decide whether to return the file-backed buffer. Skip it when it's too + // small (fragmenting memory) or page-aligned (missing a null-terminator). + static unsigned PageSize = sys::Process::getPageSizeEstimate(); + if (MaybeContent && Settings.WriteThrough.Forward && + BufferedContent->size() >= + OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer && + (BufferedContent->size() & (PageSize - 1)) != 0) { + // Synchronize the mapping to disk. + Mapping->sync(); + + // Forward the buffer. + *MaybeContent = + ContentBuffer(convertMappedFileToBuffer(std::move(Mapping))); + } + // Else, Mapping to destroy itself. + } else if (OS) { + // Write out the content and close the stream. + assert(BufferedContent && "Need to write content to a stream"); + OS->write(BufferedContent->begin(), BufferedContent->size()); + OS = nullptr; + } else { + assert(!BufferedContent && + "Content in memory with no way to write it to disk?"); + } + + // Return early if there's no temporary path. + if (!TempPath) + return Error::success(); + + // Move temporary to the final output path. + std::error_code EC = sys::fs::rename(*TempPath, getPath()); + if (!EC) + return Error::success(); + (void)sys::fs::remove(*TempPath); + return createOnDiskOutputRenameTempError(*TempPath, getPath(), EC); +} + +OnDiskOutputFile::~OnDiskOutputFile() { + if (!isOpen()) + return; + + destroyOS(); + + // If there's no FD we haven't created a file. + if (!FD) + return; + + // If there's a temporary file, that's the one to delete. + if (TempPath) + sys::fs::remove(*TempPath); + else + sys::fs::remove(getPath()); +} + +Expected> +OnDiskOutputBackend::createFileImpl(StringRef ResolvedPath, + OutputConfig Config) { + auto File = + std::make_unique(ResolvedPath, Config, Settings); + if (Error E = File->initializeFile()) + return std::move(E); + return File; +} + +Expected> +OnDiskOutputBackend::createDirectoryImpl(StringRef ResolvedPath, + OutputConfig Config) { + bool IgnoreExisting = !Config.test(OutputConfigFlag::NoOverwrite); + if (std::error_code EC = + Config.test(OutputConfigFlag::NoImplyCreateDirectories) + ? sys::fs::create_directory(ResolvedPath, IgnoreExisting) + : sys::fs::create_directories(ResolvedPath, IgnoreExisting)) + return errorCodeToError(EC); + return getDirectoryImpl(ResolvedPath); +} + +Expected> +OnDiskOutputBackend::createUniqueFileImpl(StringRef ResolvedModel, + OutputConfig Config) { + SmallString<128> Path; + int FD = -1; + { + std::error_code EC = sys::fs::createUniqueFile(ResolvedModel, FD, Path); + if (!EC) + return std::make_unique(FD, Path, Config, Settings); + + if (EC != std::errc::no_such_file_or_directory || + Config.test(OutputConfigFlag::NoImplyCreateDirectories)) + return errorCodeToError(EC); + } + + // Create the parent path and try again. + if (std::error_code EC = + sys::fs::create_directories(sys::path::parent_path(ResolvedModel))) + return errorCodeToError(EC); + if (std::error_code EC = sys::fs::createUniqueFile(ResolvedModel, FD, Path)) + return errorCodeToError(EC); + return std::make_unique(FD, Path, Config, Settings); +} + +Expected> +OnDiskOutputBackend::createUniqueDirectoryImpl(StringRef ResolvedPrefix, + OutputConfig Config) { + SmallString<128> Path; + { + std::error_code EC = sys::fs::createUniqueDirectory(ResolvedPrefix, Path); + if (!EC) + return getDirectoryImpl(Path); + + if (EC != std::errc::no_such_file_or_directory || + Config.test(OutputConfigFlag::NoImplyCreateDirectories)) + return errorCodeToError(EC); + } + + // Create the parent path and try again. + if (std::error_code EC = + sys::fs::create_directories(sys::path::parent_path(ResolvedPrefix))) + return errorCodeToError(EC); + if (std::error_code EC = sys::fs::createUniqueDirectory(ResolvedPrefix, Path)) + return errorCodeToError(EC); + return getDirectoryImpl(Path); +} + +namespace { +class InMemoryOutputDestination : public OutputFile { +public: + Error storeContentBuffer(ContentBuffer &Content) override; + + InMemoryOutputDestination(StringRef Path, + IntrusiveRefCntPtr FS, + InMemoryOutputBackend::OutputSettings Settings) + : OutputFile(Path), FS(std::move(FS)), Settings(Settings) {} + +private: + IntrusiveRefCntPtr FS; + InMemoryOutputBackend::OutputSettings Settings; +}; +} // anonymous namespace + +Error InMemoryOutputDestination::storeContentBuffer(ContentBuffer &Content) { + assert(Content.getBytes().end()[0] == 0 && "Expected null-terminated buffer"); + + // Check the content of any existing buffer. We cannot rely on the logic in + // InMemoryFileSystem::addFile, since Content and/or other backends could + // have dangling references to the buffer and addFile will destroy it if it + // can't be added. + // + // FIXME: Add an API to InMemoryFileSystem to get out an + // Optional to avoid an unnecessary malloc of MemoryBuffer. + if (ErrorOr> ExistingBuffer = + FS->getBufferForFile(getPath())) + return ExistingBuffer->get()->getBuffer() == Content.getBytes() + ? Error::success() + : createCannotOverwriteExistingOutputError(getPath()); + + std::unique_ptr Buffer = + Settings.CopyUnownedBuffers ? Content.takeOwnedBufferOrCopy(getPath()) + : Content.takeBuffer(getPath()); + bool WasAdded = FS->addFile(getPath(), 0, std::move(Buffer)); + assert(WasAdded && "Already checked this would work; what changed?"); + return Error::success(); +} + +Expected> +InMemoryOutputBackend::createFileImpl(StringRef ResolvedPath, + OutputConfig Config) { + if ((Settings.FailIfExists || Config.test(OutputConfigFlag::NoOverwrite)) && + FS->exists(ResolvedPath)) + return createCannotOverwriteExistingOutputError(ResolvedPath); + + // FIXME: OutputConfigFlag::NoOverwrite also needs to check for in-flight + // outputs. + + return std::make_unique(ResolvedPath, FS, + Settings); +} + +namespace { +class MirroringOutputBackend : public OutputBackend { + class File; + +public: + Expected + resolveOutputPath(const Twine &Path, + SmallVectorImpl &PathStorage) const override { + return Backend1->resolveOutputPath(Path, PathStorage); + } + + Expected> + createFileImpl(StringRef ResolvedPath, OutputConfig Config) override; + + Expected> + createUniqueFileImpl(StringRef ResolvedModel, OutputConfig Config) override; + + Expected> + createDirectoryImpl(StringRef ResolvedPath, OutputConfig Config) override; + + Expected> + createUniqueDirectoryImpl(StringRef ResolvedModel, + OutputConfig Config) override; + + MirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) + : OutputBackend(Backend1->getPathStyle()), Backend1(std::move(Backend1)), + Backend2(std::move(Backend2)) { + assert(this->Backend2->getPathStyle() == getPathStyle() && + "Path style needs to match between mirrored backends"); + } + +private: + IntrusiveRefCntPtr Backend1; + IntrusiveRefCntPtr Backend2; +}; +} // anonymous namespace + +class MirroringOutputBackend::File : public OutputFile { + Error storeContentBuffer(ContentBuffer &Content) override { + if (Error E = storeContentBufferIn(Content, *File1)) + return E; + return File2->storeContentBufferIn(Content, *File2); + } + +public: + File(std::unique_ptr File1, std::unique_ptr File2) + : OutputFile(File1->getPath()), File1(std::move(File1)), + File2(std::move(File2)) {} + + static std::unique_ptr create(std::unique_ptr File1, + std::unique_ptr File2) { + if (isNull(*File1)) + return File2; + if (isNull(*File2)) + return File1; + return std::make_unique(std::move(File1), std::move(File2)); + } + + std::unique_ptr File1; + std::unique_ptr File2; +}; + +IntrusiveRefCntPtr llvm::vfs::makeMirroringOutputBackend( + IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) { + return std::make_unique(std::move(Backend1), + std::move(Backend2)); +} + +Expected> +MirroringOutputBackend::createFileImpl(StringRef ResolvedPath, + OutputConfig Config) { + // Create destinations for each backend. + SmallVector, 2> Files; + for (OutputBackend *B : {&*Backend1, &*Backend2}) { + Expected> File = + B->createFile(ResolvedPath, Config); + if (!File) + return File.takeError(); + Files.push_back(std::move(*File)); + } + + return File::create(std::move(Files[0]), std::move(Files[1])); +} + +Expected> +MirroringOutputBackend::createUniqueFileImpl(StringRef ResolvedModel, + OutputConfig Config) { + // Create a unique file in the first backend. + Expected> ExpectedFile = + Backend1->createUniqueFile(ResolvedModel, Config); + if (!ExpectedFile) + return ExpectedFile.takeError(); + std::unique_ptr File = std::move(*ExpectedFile); + + // Create the same file in the second backend. + Config.set(OutputConfigFlag::NoOverwrite); + ExpectedFile = Backend2->createFile(File->getPath(), Config); + if (!ExpectedFile) + return ExpectedFile.takeError(); + return File::create(std::move(File), std::move(*ExpectedFile)); +} + +Expected> +MirroringOutputBackend::createDirectoryImpl(StringRef ResolvedPath, + OutputConfig Config) { + // Create directories for each backend in order to check for errors. + for (OutputBackend *B : {&*Backend1, &*Backend2}) { + Expected> Dir = + B->createDirectory(ResolvedPath, Config); + if (!Dir) + return Dir.takeError(); + } + + // Return an output directory for this backend. + return makeIntrusiveRefCnt(this, ResolvedPath); +} + +Expected> +MirroringOutputBackend::createUniqueDirectoryImpl(StringRef ResolvedModel, + OutputConfig Config) { + // Create a unique directory in the first backend. + Expected> ExpectedDir = + Backend1->createUniqueDirectory(ResolvedModel, Config); + if (!ExpectedDir) + return ExpectedDir.takeError(); + IntrusiveRefCntPtr Dir = std::move(*ExpectedDir); + + // Create the same directory in the second backend. + Config.set(OutputConfigFlag::NoOverwrite); + ExpectedDir = Backend2->createDirectory(Dir->getPath(), Config); + if (!ExpectedDir) + return ExpectedDir.takeError(); + return makeIntrusiveRefCnt(this, Dir->getPath()); +} + +namespace { +class FilteringOutputBackend : public ProxyOutputBackend { +public: + Expected> + createFileImpl(StringRef ResolvedPath, OutputConfig Config) override { + if (!Filter(ResolvedPath, Config)) + return makeNullOutput(ResolvedPath); + return ProxyOutputBackend::createFileImpl(ResolvedPath, Config); + } + + using FilterType = unique_function; + FilteringOutputBackend(IntrusiveRefCntPtr UnderlyingBackend, + FilterType Filter) + : ProxyOutputBackend(std::move(UnderlyingBackend)), + Filter(std::move(Filter)) {} + +private: + FilterType Filter; +}; +} // anonymous namespace + +IntrusiveRefCntPtr llvm::vfs::makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + FilteringOutputBackend::FilterType Filter) { + return std::make_unique(std::move(UnderlyingBackend), + std::move(Filter)); +} Index: llvm/unittests/Support/CMakeLists.txt =================================================================== --- llvm/unittests/Support/CMakeLists.txt +++ llvm/unittests/Support/CMakeLists.txt @@ -58,6 +58,8 @@ MemoryTest.cpp NativeFormatTests.cpp OptimizedStructLayoutTest.cpp + OutputConfigTest.cpp + OutputBackendTest.cpp ParallelTest.cpp Path.cpp ProcessTest.cpp Index: llvm/unittests/Support/OutputBackendTest.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/OutputBackendTest.cpp @@ -0,0 +1,1220 @@ +//===- unittests/Support/OutputBackendTest.cpp - OutputBackend tests ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/OutputBackend.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +using ContentBuffer = OutputFile::ContentBuffer; + +TEST(OutputFileContentBufferTest, takeBuffer) { + StringRef S = "data"; + ContentBuffer Content = S; + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(S.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeBuffer("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + + // The buffer should be a reference to the original data, and Content should + // still have it too. + EXPECT_EQ(S.begin(), Taken->getBufferStart()); + EXPECT_EQ(S.begin(), Content.getBytes().begin()); +} + +TEST(OutputFileContentBufferTest, takeBufferVector) { + SmallString<0> V = StringRef("data"); + StringRef R = V; + + ContentBuffer Content = std::move(V); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.begin(), Content.getBytes().begin()); + EXPECT_NE(V.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeBuffer("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.begin(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); +} + +TEST(OutputFileContentBufferTest, takeBufferVectorSmallStorage) { + // Something using small storage will have to move. + SmallString<128> V = StringRef("data"); + ASSERT_EQ(128u, V.capacity()); + StringRef R = V; + + ContentBuffer Content = std::move(V); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_NE(R.begin(), Content.getBytes().begin()); +} + +TEST(OutputFileContentBufferTest, takeBufferMemoryBuffer) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeBuffer("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); +} + +TEST(OutputFileContentBufferTest, takeBufferMemoryBufferSameName) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data", "name"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + // This exact buffer should be returned since the name matches. + std::unique_ptr Taken = Content.takeBuffer("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + EXPECT_EQ(R.getBufferIdentifier().begin(), + Taken->getBufferIdentifier().begin()); +} + +TEST(OutputFileContentBufferTest, takeBufferMemoryBufferWrongName) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data", "other"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_TRUE(Content.ownsContent()); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + // Check that the name is updated. + std::unique_ptr Taken = Content.takeBuffer("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrNull) { + StringRef S = "data"; + ContentBuffer Content = S; + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(S.begin(), Content.getBytes().begin()); + + // ContentBuffer doesn't own this. + EXPECT_FALSE(Content.takeOwnedBufferOrNull("name")); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrNullVector) { + SmallString<0> V = StringRef("data"); + StringRef R = V; + + ContentBuffer Content = std::move(V); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.begin(), Content.getBytes().begin()); + EXPECT_NE(V.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrNull("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.begin(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_FALSE(Content.ownsContent()); + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + + // The next call should fail since it's no longer owned. + EXPECT_FALSE(Content.takeOwnedBufferOrNull("name")); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrNullVectorEmpty) { + SmallString<0> V; + + ContentBuffer Content = std::move(V); + EXPECT_EQ("", Content.getBytes()); + + // Check that ContentBuffer::takeOwnedBufferOrNull() fails for an empty + // vector. + EXPECT_FALSE(Content.ownsContent()); + ASSERT_FALSE(Content.takeOwnedBufferOrNull("name")); + + // ContentBuffer::takeBuffer() should still work. + std::unique_ptr Taken = Content.takeBuffer("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrNullMemoryBuffer) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrNull("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + + // The next call should fail since it's no longer owned. + EXPECT_FALSE(Content.takeOwnedBufferOrNull("name")); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrCopy) { + StringRef S = "data"; + ContentBuffer Content = S; + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(S.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrCopy("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + + // We should have a copy, but Content should be unchanged. + EXPECT_NE(S.begin(), Taken->getBufferStart()); + EXPECT_EQ(S.begin(), Content.getBytes().begin()); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrCopyVector) { + SmallString<0> V = StringRef("data"); + StringRef R = V; + + ContentBuffer Content = std::move(V); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.begin(), Content.getBytes().begin()); + EXPECT_NE(V.begin(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrCopy("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + + // Content should still have a reference to the data, but it's no longer + // owned. + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + EXPECT_FALSE(Content.takeOwnedBufferOrNull("name")); + + // The next call should get a different (but equal) buffer. + std::unique_ptr Copy = Content.takeOwnedBufferOrCopy("name"); + ASSERT_TRUE(Copy); + EXPECT_EQ("data", Copy->getBuffer()); + EXPECT_EQ("name", Copy->getBufferIdentifier()); + EXPECT_NE(Taken->getBufferStart(), Copy->getBufferStart()); +} + +TEST(OutputFileContentBufferTest, takeOwnedBufferOrCopyMemoryBuffer) { + std::unique_ptr B = MemoryBuffer::getMemBuffer("data"); + MemoryBufferRef R = *B; + + ContentBuffer Content = std::move(B); + EXPECT_EQ("data", Content.getBytes()); + EXPECT_EQ(R.getBufferStart(), Content.getBytes().begin()); + + std::unique_ptr Taken = Content.takeOwnedBufferOrCopy("name"); + ASSERT_TRUE(Taken); + EXPECT_EQ("data", Taken->getBuffer()); + EXPECT_EQ("name", Taken->getBufferIdentifier()); + EXPECT_EQ(R.getBufferStart(), Taken->getBufferStart()); + + // Content should still have a reference to the data. + EXPECT_EQ(Taken->getBuffer(), Content.getBytes()); + EXPECT_EQ(Taken->getBufferStart(), Content.getBytes().begin()); + EXPECT_EQ(Content.getBytes().begin(), Taken->getBufferStart()); + + // The next call should get a different (but equal) buffer. + std::unique_ptr Copy = Content.takeOwnedBufferOrCopy("name"); + ASSERT_TRUE(Copy); + EXPECT_EQ("data", Copy->getBuffer()); + EXPECT_EQ("name", Copy->getBufferIdentifier()); + EXPECT_NE(Taken->getBufferStart(), Copy->getBufferStart()); +} + +struct OnDiskTempDirectory { + bool Created = false; + SmallString<128> Path; + + OnDiskTempDirectory() = delete; + explicit OnDiskTempDirectory(StringRef Name) { + if (!sys::fs::createUniqueDirectory(Name, Path)) + Created = true; + } + ~OnDiskTempDirectory() { + if (Created) + sys::fs::remove_directories(Path); + } +}; + +struct OnDiskFile { + const OnDiskTempDirectory &D; + SmallString<128> Path; + StringRef ParentPath; + StringRef Filename; + StringRef Stem; + StringRef Extension; + std::unique_ptr LastBuffer; + + template + OnDiskFile(const OnDiskTempDirectory &D, PathArgTypes &&... PathArgs) : D(D) { + assert(D.Created); + sys::path::append(Path, D.Path, std::forward(PathArgs)...); + ParentPath = sys::path::parent_path(Path); + Filename = sys::path::filename(Path); + Stem = sys::path::stem(Filename); + Extension = sys::path::extension(Filename); + } + + Optional findTemp() const { + std::error_code EC; + for (sys::fs::directory_iterator I(ParentPath, EC), E; !EC && I != E; + I.increment(EC)) { + StringRef TempPath = I->path(); + if (!TempPath.startswith(D.Path)) + continue; + + // Look for "-*..tmp". + if (sys::path::extension(TempPath) != ".tmp") + continue; + + // Drop the ".tmp" and check the extension and stem. + StringRef TempStem = sys::path::stem(TempPath); + if (sys::path::extension(TempStem) != Extension) + continue; + StringRef OriginalStem = sys::path::stem(TempStem); + if (!OriginalStem.startswith(Stem)) + continue; + if (!OriginalStem.drop_front(Stem.size()).startswith("-")) + continue; + + // Found it. + return OnDiskFile(D, TempPath.drop_front(D.Path.size() + 1)); + } + return None; + } + + Optional getCurrentUniqueID() { + sys::fs::file_status Status; + sys::fs::status(Path, Status, /*follow=*/false); + if (!sys::fs::is_regular_file(Status)) + return None; + return Status.getUniqueID(); + } + + bool hasUniqueID(sys::fs::UniqueID ID) { + auto CurrentID = getCurrentUniqueID(); + if (!CurrentID) + return false; + return *CurrentID == ID; + } + + Optional getCurrentContent() { + auto OnDiskOrErr = MemoryBuffer::getFile(Path); + if (!OnDiskOrErr) + return None; + LastBuffer = std::move(*OnDiskOrErr); + return LastBuffer->getBuffer(); + } + + bool equalsCurrentContent(StringRef Data) { + auto CurrentContent = getCurrentContent(); + if (!CurrentContent) + return false; + return *CurrentContent == Data; + } + + bool equalsCurrentContent(NoneType) { return getCurrentContent() == None; } +}; + +struct OutputBackendTest : public ::testing::Test { + Optional D; + + void TearDown() override { D = None; } + + bool makeTempDirectory() { + D.emplace("OutputBackendTest.d"); + return D->Created; + } + + template + std::unique_ptr + expectedToPointer(Expected> ExpectedPointer) { + if (ExpectedPointer) + return std::move(*ExpectedPointer); + consumeError(ExpectedPointer.takeError()); + return nullptr; + } + + template + IntrusiveRefCntPtr + expectedToPointer(Expected> ExpectedPointer) { + if (ExpectedPointer) + return std::move(*ExpectedPointer); + consumeError(ExpectedPointer.takeError()); + return nullptr; + } + + std::unique_ptr openBufferForRead(InMemoryFileSystem &FS, + StringRef Path) { + if (auto FileOrErr = FS.openFileForRead(Path)) + if (*FileOrErr) + if (auto BufferOrErr = (*FileOrErr)->getBuffer(Path)) + if (*BufferOrErr) + return std::move(*BufferOrErr); + return nullptr; + } +}; + +TEST_F(OutputBackendTest, Null) { + auto Backend = makeNullOutputBackend(); + + // Create the output. + auto O = expectedToPointer(Backend->createFile("ignored.data")); + ASSERT_TRUE(O); + ASSERT_TRUE(O->getOS()); + + // Write some data into it and close the stream. + *O->takeOS() << "some data"; + + // Closing should have no effect. + EXPECT_FALSE(errorToBool(O->close())); +} + +TEST_F(OutputBackendTest, NullErase) { + auto Backend = makeNullOutputBackend(); + + // Create the output. + auto O = expectedToPointer(Backend->createFile("ignored.data")); + ASSERT_TRUE(O); + ASSERT_TRUE(O->getOS()); + + // Write some data into it and close the stream. + *O->takeOS() << "some data"; + + // Erasing should have no effect. + O = nullptr; +} + +TEST_F(OutputBackendTest, OnDisk) { + auto Backend = makeIntrusiveRefCnt(); + Backend->getSettings().WriteThrough.Use = false; + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + Optional TempID = Temp->getCurrentUniqueID(); + ASSERT_TRUE(TempID); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(Temp->equalsCurrentContent("some data")); + EXPECT_TRUE(File.equalsCurrentContent(None)); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // The temp file should have been moved. + EXPECT_TRUE(File.hasUniqueID(*TempID)); +} + +TEST_F(OutputBackendTest, OnDiskErase) { + auto Backend = makeIntrusiveRefCnt(); + Backend->getSettings().WriteThrough.Use = false; + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + + // Write some data into it and close the stream. + *O->takeOS() << "some data"; + + // Erase and check that the temp is gone. + O = nullptr; + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent(None)); +} + +TEST_F(OutputBackendTest, OnDiskMissingDirectories) { + auto Backend = makeIntrusiveRefCnt(); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "missing", "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + Optional TempID = Temp->getCurrentUniqueID(); + ASSERT_TRUE(TempID); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(Temp->equalsCurrentContent("some data")); + EXPECT_TRUE(File.equalsCurrentContent(None)); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // The temp file should have been moved. + EXPECT_TRUE(File.hasUniqueID(*TempID)); +} + +TEST_F(OutputBackendTest, OnDiskNoImplyCreateDirectories) { + auto Backend = makeIntrusiveRefCnt(); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "missing", "on-disk.data"); + + // Fail to create the missing directory. + auto Expected = Backend->createFile( + File.Path, {OutputConfigFlag::NoImplyCreateDirectories}); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value()); +} + +TEST_F(OutputBackendTest, OnDiskNoAtomicWrite) { + auto Backend = makeIntrusiveRefCnt(); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer( + Backend->createFile(File.Path, {OutputConfigFlag::NoAtomicWrite})); + ASSERT_TRUE(O); + + // File should exist with no temporary. + EXPECT_TRUE(File.equalsCurrentContent("")); + Optional ID = File.getCurrentUniqueID(); + ASSERT_TRUE(ID); + Optional Temp = File.findTemp(); + ASSERT_FALSE(Temp); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + EXPECT_TRUE(File.hasUniqueID(*ID)); +} + +TEST_F(OutputBackendTest, OnDiskNoAtomicWriteErase) { + auto Backend = makeIntrusiveRefCnt(); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer( + Backend->createFile(File.Path, {OutputConfigFlag::NoAtomicWrite})); + ASSERT_TRUE(O); + + // File should exist with no temporary. + EXPECT_TRUE(File.equalsCurrentContent("")); + + // Write some data into it and flush. + *O->takeOS() << "some data"; + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // Erase. Since NoAtomicWrite was used the file should now be gone. + O = nullptr; +} + +TEST_F(OutputBackendTest, OnDiskWriteThroughUse) { + auto Backend = makeIntrusiveRefCnt(); + Backend->getSettings().WriteThrough.Use = true; + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // File shouldn't exist, but the temporary should. + EXPECT_TRUE(File.equalsCurrentContent(None)); + Optional Temp = File.findTemp(); + ASSERT_TRUE(Temp); + Optional TempID = Temp->getCurrentUniqueID(); + ASSERT_TRUE(TempID); + + // Write some data into it and flush. Confirm it's not on disk because it's + // waiting to be written using a write-through buffer. + *O->takeOS() << "some data"; + EXPECT_TRUE(Temp->equalsCurrentContent("")); + EXPECT_TRUE(File.equalsCurrentContent(None)); + + // Close and check again. + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(Temp->equalsCurrentContent(None)); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + EXPECT_TRUE(File.hasUniqueID(*TempID)); +} + +struct CaptureLastBufferOutputBackend : public StableUniqueEntityAdaptor<> { + struct File : public OutputFile { + Error storeContentBuffer(ContentBuffer &Content) final { + Storage = Content.takeBuffer(getPath()); + return Error::success(); + } + + File(StringRef Path, std::unique_ptr &Storage) + : OutputFile(Path), Storage(Storage) {} + + std::unique_ptr &Storage; + }; + + Expected> createFileImpl(StringRef Path, + OutputConfig) final { + return std::make_unique(Path, Storage); + } + + std::unique_ptr &Storage; + CaptureLastBufferOutputBackend(std::unique_ptr &Storage) + : StableUniqueEntityAdaptorType(sys::path::Style::native), + Storage(Storage) {} +}; + +TEST_F(OutputBackendTest, OnDiskWriteThroughForward) { + auto OnDiskBackend = makeIntrusiveRefCnt(); + OnDiskBackend->getSettings().WriteThrough.Use = true; + OnDiskBackend->getSettings().WriteThrough.Forward = true; + + std::unique_ptr Buffer; + auto Backend = makeMirroringOutputBackend( + std::move(OnDiskBackend), + makeIntrusiveRefCnt(Buffer)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write enough data to get a write-through buffer, and add one to ensure + // it's not page-aligned. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer + 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Check that the saved buffer is a write-through buffer. + EXPECT_TRUE(Buffer); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); + EXPECT_EQ(MemoryBuffer::MemoryBuffer_MMap, Buffer->getBufferKind()); +} + +TEST_F(OutputBackendTest, OnDiskWriteThroughForwardTooSmall) { + auto OnDiskBackend = makeIntrusiveRefCnt(); + OnDiskBackend->getSettings().WriteThrough.Use = true; + OnDiskBackend->getSettings().WriteThrough.Forward = true; + + std::unique_ptr Buffer; + auto Backend = makeMirroringOutputBackend( + std::move(OnDiskBackend), + makeIntrusiveRefCnt(Buffer)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write almost enough data to return write-through buffer, but not quite. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer - 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Check that the saved buffer is not write-through buffer. + EXPECT_TRUE(Buffer); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); + EXPECT_EQ(MemoryBuffer::MemoryBuffer_Malloc, Buffer->getBufferKind()); +} + +TEST_F(OutputBackendTest, FilteredOnDisk) { + auto Backend = + makeFilteringOutputBackend(makeIntrusiveRefCnt(), + [](StringRef, OutputConfig) { return true; }); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. The content should be there. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent("some data")); +} + +TEST_F(OutputBackendTest, FilteredOnDiskSkipped) { + auto Backend = + makeFilteringOutputBackend(makeIntrusiveRefCnt(), + [](StringRef, OutputConfig) { return false; }); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. It should not exist on disk. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(None)); +} + +TEST_F(OutputBackendTest, InMemory) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeIntrusiveRefCnt(FS); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + ASSERT_FALSE(FS->exists(Path)); + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + ASSERT_FALSE(FS->exists(Path)); + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS->exists(Path)); + std::unique_ptr Buffer = openBufferForRead(*FS, Path); + EXPECT_EQ(Path, Buffer->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer->getBuffer()); +} + +TEST_F(OutputBackendTest, InMemoryFailIfExists) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeIntrusiveRefCnt(FS); + + // Add a conflicting file to FS. + StringRef Path = "//root/in/memory.data"; + FS->addFile(Path, 0, MemoryBuffer::getMemBuffer("some data")); + + // Check that we get the error from failing to clobber the in-memory file. + Backend->getSettings().FailIfExists = true; + auto Expected = Backend->createFile(Path); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::file_exists), EC.value()); + + // No error if we turn off the flag though. + Backend->getSettings().FailIfExists = false; + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Writing identical data should be okay. + *O->takeOS() << "some data"; + ASSERT_FALSE(errorToBool(O->close())); +} + +TEST_F(OutputBackendTest, InMemoryMismatchedContent) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeIntrusiveRefCnt(FS); + Backend->getSettings().FailIfExists = false; + + // Create the output. + StringRef Path = "//root/in/memory.data"; + ASSERT_FALSE(FS->exists(Path)); + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_FALSE(FS->exists(Path)); + ASSERT_TRUE(O); + + // Add a conflicting file to FS. + FS->addFile(Path, 0, MemoryBuffer::getMemBuffer("old data")); + ASSERT_TRUE(FS->exists(Path)); + + // Write some data into it, flush, and close. + *O->takeOS() << "new data"; + Error E = O->close(); + + // Check the error. + ASSERT_TRUE(bool(E)); + std::error_code EC = errorToErrorCode(std::move(E)); + EXPECT_EQ(int(std::errc::file_exists), EC.value()); +} + +TEST_F(OutputBackendTest, InMemoryNoOverwrite) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeIntrusiveRefCnt(FS); + Backend->getSettings().FailIfExists = false; + + // Create the output once. This should be fine. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer( + Backend->createFile(Path, {OutputConfigFlag::NoOverwrite})); + ASSERT_TRUE(O); + *O->takeOS() << "some data"; + ASSERT_FALSE(errorToBool(O->close())); + + // Doing it again should fail. + auto Expected = Backend->createFile(Path, {OutputConfigFlag::NoOverwrite}); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::file_exists), EC.value()); + + // It should fail even in a different backend. + Backend = makeIntrusiveRefCnt(FS); + Backend->getSettings().FailIfExists = false; + Expected = Backend->createFile(Path, {OutputConfigFlag::NoOverwrite}); + ASSERT_FALSE(Expected); + EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::file_exists), EC.value()); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemory) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent("some data")); + + // Lookup the file in FS. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer->getBuffer()); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemoryFailIfExists) { + auto FS = makeIntrusiveRefCnt(); + auto InMemoryBackend = makeIntrusiveRefCnt(FS); + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(), InMemoryBackend); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Add a file with the same name to FS. + FS->addFile(File.Path, 0, MemoryBuffer::getMemBuffer("some data")); + + // Check that we get the error from failing to clobber the in-memory file. + InMemoryBackend->getSettings().FailIfExists = true; + auto Expected = Backend->createFile(File.Path); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::file_exists), EC.value()); + + // No error if we turn off the flag though. + InMemoryBackend->getSettings().FailIfExists = false; + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Writing identical data should be okay. + *O->takeOS() << "some data"; + ASSERT_FALSE(errorToBool(O->close())); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemoryMismatchedContent) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the output. + ASSERT_FALSE(FS->exists(File.Path)); + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_FALSE(FS->exists(File.Path)); + ASSERT_TRUE(O); + + // Add a conflicting file to FS. + FS->addFile(File.Path, 0, MemoryBuffer::getMemBuffer("old data")); + ASSERT_TRUE(FS->exists(File.Path)); + + // Write some data into it, flush, and close. + *O->takeOS() << "new data"; + Error E = O->close(); + + // Check the error. + ASSERT_TRUE(bool(E)); + std::error_code EC = errorToErrorCode(std::move(E)); + EXPECT_EQ(int(std::errc::file_exists), EC.value()); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemoryNoImplyCreateDirectories) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(), + makeIntrusiveRefCnt(FS)); + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "missing", "on-disk.data"); + + // Check that we get the error from failing to create the missing directory + // on-disk. + auto Expected = Backend->createFile( + File.Path, {OutputConfigFlag::NoImplyCreateDirectories}); + ASSERT_FALSE(Expected); + std::error_code EC = errorToErrorCode(Expected.takeError()); + EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value()); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemoryWriteThrough) { + auto FS = makeIntrusiveRefCnt(); + auto OnDiskBackend = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + OnDiskBackend, makeIntrusiveRefCnt(FS)); + OnDiskBackend->getSettings().WriteThrough.Use = true; + OnDiskBackend->getSettings().WriteThrough.Forward = true; + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write enough data to get a write-through buffer, and add one to ensure + // it's not page-aligned. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer + 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Lookup the file in FS. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemoryWriteThroughPageAligned) { + auto FS = makeIntrusiveRefCnt(); + auto OnDiskBackend = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + OnDiskBackend, makeIntrusiveRefCnt(FS)); + OnDiskBackend->getSettings().WriteThrough.Use = true; + OnDiskBackend->getSettings().WriteThrough.Forward = true; + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write just enough data to get a write-through buffer. It'll likely be + // page-aligned. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer, 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Lookup the file in FS. Ensure no errors when it creates a null-terminated + // reference to the buffer. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); +} + +TEST_F(OutputBackendTest, MirrorOnDiskInMemoryWriteThroughTooSmall) { + auto FS = makeIntrusiveRefCnt(); + auto OnDiskBackend = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + OnDiskBackend, makeIntrusiveRefCnt(FS)); + OnDiskBackend->getSettings().WriteThrough.Use = true; + OnDiskBackend->getSettings().WriteThrough.Forward = true; + + // Get a filename we can use on disk. + ASSERT_TRUE(makeTempDirectory()); + OnDiskFile File(*D, "on-disk.data"); + + // Create the file. + auto O = expectedToPointer(Backend->createFile(File.Path)); + ASSERT_TRUE(O); + + // Write almost (but not quite) enough data to get a write-through buffer. + std::string Data; + Data.append(OnDiskOutputBackend::MinimumSizeToReturnWriteThroughBuffer - 1, + 'z'); + *O->takeOS() << Data; + EXPECT_FALSE(errorToBool(O->close())); + EXPECT_TRUE(File.equalsCurrentContent(Data)); + + // Lookup the file in FS. + std::unique_ptr Buffer = openBufferForRead(*FS, File.Path); + EXPECT_EQ(File.Path, Buffer->getBufferIdentifier()); + EXPECT_EQ(Data, Buffer->getBuffer()); +} + +TEST_F(OutputBackendTest, FilteredInMemory) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = + makeFilteringOutputBackend(makeIntrusiveRefCnt(FS), + [](StringRef, OutputConfig) { return true; }); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS->exists(Path)); + std::unique_ptr Buffer = openBufferForRead(*FS, Path); + EXPECT_EQ(Path, Buffer->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer->getBuffer()); +} + +TEST_F(OutputBackendTest, FilteredInMemorySkipped) { + auto FS = makeIntrusiveRefCnt(); + auto Backend = + makeFilteringOutputBackend(makeIntrusiveRefCnt(FS), + [](StringRef, OutputConfig) { return false; }); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + ASSERT_FALSE(FS->exists(Path)); + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // The file should have been filtered out. + ASSERT_FALSE(FS->exists(Path)); +} + +TEST_F(OutputBackendTest, MirrorFilteredInMemoryInMemoryNotFiltered) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + makeFilteringOutputBackend( + makeIntrusiveRefCnt(FS1), + [](StringRef, OutputConfig) { return false; }), + makeIntrusiveRefCnt(FS2)); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. It should only be skipped in FS1. + EXPECT_TRUE(!FS1->exists(Path)); + ASSERT_TRUE(FS2->exists(Path)); + std::unique_ptr Buffer2 = openBufferForRead(*FS2, Path); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer2->getBuffer()); +} + +TEST_F(OutputBackendTest, MirrorInMemoryWithFilteredInMemory) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(FS1), + makeFilteringOutputBackend( + makeIntrusiveRefCnt(FS2), + [](StringRef, OutputConfig) { return false; })); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. It should only be skipped in FS1. + EXPECT_TRUE(!FS2->exists(Path)); + ASSERT_TRUE(FS1->exists(Path)); + std::unique_ptr Buffer2 = openBufferForRead(*FS1, Path); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer2->getBuffer()); +} + +TEST_F(OutputBackendTest, MirrorInMemoryInMemory) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(FS1), + makeIntrusiveRefCnt(FS2)); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS1->exists(Path)); + ASSERT_TRUE(FS2->exists(Path)); + std::unique_ptr Buffer1 = openBufferForRead(*FS1, Path); + std::unique_ptr Buffer2 = openBufferForRead(*FS2, Path); + ASSERT_TRUE(Buffer1); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer1->getBufferIdentifier()); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer1->getBuffer()); + EXPECT_EQ("some data", Buffer2->getBuffer()); + + // Should be a reference to the same memory. + EXPECT_EQ(Buffer1->getBufferStart(), Buffer2->getBufferStart()); +} + +TEST_F(OutputBackendTest, MirrorInMemoryInMemoryOwnBufferForMirror) { + auto FS1 = makeIntrusiveRefCnt(); + auto FS2 = makeIntrusiveRefCnt(); + auto CopyingBackend = makeIntrusiveRefCnt(FS2); + CopyingBackend->getSettings().CopyUnownedBuffers = true; + auto Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(FS1), + std::move(CopyingBackend)); + + // Create the output. + StringRef Path = "//root/in/memory.data"; + auto O = expectedToPointer(Backend->createFile(Path)); + ASSERT_TRUE(O); + + // Write some data into it, flush, and close. + *O->takeOS() << "some data"; + EXPECT_FALSE(errorToBool(O->close())); + + // Lookup the file. + ASSERT_TRUE(FS1->exists(Path)); + ASSERT_TRUE(FS2->exists(Path)); + std::unique_ptr Buffer1 = openBufferForRead(*FS1, Path); + std::unique_ptr Buffer2 = openBufferForRead(*FS2, Path); + ASSERT_TRUE(Buffer1); + ASSERT_TRUE(Buffer2); + EXPECT_EQ(Path, Buffer1->getBufferIdentifier()); + EXPECT_EQ(Path, Buffer2->getBufferIdentifier()); + EXPECT_EQ("some data", Buffer1->getBuffer()); + EXPECT_EQ("some data", Buffer2->getBuffer()); + + // Should be a copy. + EXPECT_NE(Buffer1->getBufferStart(), Buffer2->getBufferStart()); +} + +} // anonymous namespace Index: llvm/unittests/Support/OutputConfigTest.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/OutputConfigTest.cpp @@ -0,0 +1,110 @@ +//===- unittests/Support/OutputConfigTest.cpp - OutputConfig tests --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/OutputBackend.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +TEST(OutputConfigTest, construct) { + EXPECT_TRUE(OutputConfig().none()); + EXPECT_FALSE(OutputConfig().test(OutputConfigFlag::Text)); + EXPECT_FALSE(OutputConfig().test(OutputConfigFlag::NoCrashCleanup)); + + { + OutputConfig Config = {OutputConfigFlag::NoCrashCleanup}; + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(OutputConfigFlag::Text)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } + + { + OutputConfig Config = {OutputConfigFlag::Text, + OutputConfigFlag::NoCrashCleanup}; + EXPECT_FALSE(Config.none()); + EXPECT_TRUE(Config.test(OutputConfigFlag::Text)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } +} + +TEST(OutputConfigTest, set) { + { + // Set one. + OutputConfig Config; + Config.set(OutputConfigFlag::Text); + EXPECT_FALSE(Config.none()); + EXPECT_TRUE(Config.test(OutputConfigFlag::Text)); + EXPECT_FALSE(Config.test(OutputConfigFlag::NoCrashCleanup)); + + // Undo it. + Config.set(OutputConfigFlag::Text, false); + EXPECT_TRUE(Config.none()); + EXPECT_FALSE(Config.test(OutputConfigFlag::Text)); + EXPECT_FALSE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } + + { + // Set multiple. + OutputConfig Config; + Config.set({OutputConfigFlag::Text, OutputConfigFlag::NoCrashCleanup, + OutputConfigFlag::NoImplyCreateDirectories}); + EXPECT_FALSE(Config.none()); + EXPECT_TRUE(Config.test(OutputConfigFlag::Text)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoCrashCleanup)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoImplyCreateDirectories)); + + // Undo two of them. + Config.set( + {OutputConfigFlag::Text, OutputConfigFlag::NoImplyCreateDirectories}, + false); + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(OutputConfigFlag::Text)); + EXPECT_FALSE(Config.test(OutputConfigFlag::NoImplyCreateDirectories)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } +} + +TEST(OutputConfigTest, reset) { + { + // Reset one. + OutputConfig Config = {OutputConfigFlag::Text, + OutputConfigFlag::NoCrashCleanup}; + Config.reset(OutputConfigFlag::Text); + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(OutputConfigFlag::Text)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } + + { + // Reset both. + OutputConfig Config = {OutputConfigFlag::Text, + OutputConfigFlag::NoCrashCleanup}; + Config.reset({OutputConfigFlag::Text, OutputConfigFlag::NoCrashCleanup}); + EXPECT_TRUE(Config.none()); + EXPECT_FALSE(Config.test(OutputConfigFlag::Text)); + EXPECT_FALSE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } + + { + // Reset multiple (but not all). + OutputConfig Config = {OutputConfigFlag::Text, + OutputConfigFlag::NoCrashCleanup, + OutputConfigFlag::NoImplyCreateDirectories}; + Config.reset( + {OutputConfigFlag::Text, OutputConfigFlag::NoImplyCreateDirectories}); + EXPECT_FALSE(Config.none()); + EXPECT_FALSE(Config.test(OutputConfigFlag::Text)); + EXPECT_FALSE(Config.test(OutputConfigFlag::NoImplyCreateDirectories)); + EXPECT_TRUE(Config.test(OutputConfigFlag::NoCrashCleanup)); + } +} + +} // anonymous namespace