[Support] Improve Caching conformance with Support library behavior

[noajshu]

Summary

This diff makes several amendments to the local file caching mechanism which was migrated from ThinLTO to Support in rGe678c51177102845c93529d457b020f969125373 in response to follow-up discussion on that commit.

[noajshu]

Improve error handling in Caching support library.
Most of the functions now return Expected<>, except for the non-trivial destructor of CacheStream.

[lattner]

nice, thank you for improving this!

[noajshu]

Rebase against main.

clang/lib/CodeGen/BackendUtil.cpp

static void runThinLTOBackend(
   // We can simply import the values mentioned in the combined index, since
   // we should only invoke this using the individual indexes written out
   // via a WriteIndexesThinBackend.
   FunctionImporter::ImportMapTy ImportList;
   if (!lto::initImportList(*M, *CombinedIndex, ImportList))
     return;
 
   auto AddStream = [&](size_t Task) {
-    return std::make_unique<NativeObjectStream>(std::move(OS));
+    return std::make_unique<CachedFileStream>(std::move(OS));
   };
   lto::Config Conf;
   if (CGOpts.SaveTempsFilePrefix != "") {
     if (Error E = Conf.addSaveTemps(CGOpts.SaveTempsFilePrefix + ".",
                                     /* UseInputModulePath */ false)) {
       handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
         errs() << "Error setting up ThinLTO save-temps: " << EIB.message()
                << '\n';

lld/COFF/LTO.cpp

 std::vector<InputFile *> BitcodeCompiler::compile(COFFLinkerContext &ctx) {
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
   files.resize(maxTasks);
 
   // The /lldltocache option specifies the path to a directory in which to cache
   // native object files for ThinLTO incremental builds. If a path was
   // specified, configure LTO to use it as the cache directory.
-  NativeObjectCache cache;
+  FileCache cache;
   if (!config->ltoCache.empty())
     cache =
         check(localCache("ThinLTO", "Thin", config->ltoCache,
                          [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
                            files[task] = std::move(mb);
                          }));
 
   checkError(ltoObj->run(
       [&](size_t task) {
-        return std::make_unique<NativeObjectStream>(
+        return std::make_unique<CachedFileStream>(
             std::make_unique<raw_svector_ostream>(buf[task]));
       },
       cache));
 
   // Emit empty index files for non-indexed files
   for (StringRef s : thinIndices) {
     std::string path = getThinLTOOutputFile(s);
     openFile(path + ".thinlto.bc");

lld/ELF/LTO.cpp

 std::vector<InputFile *> BitcodeCompiler::compile() {
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
   files.resize(maxTasks);
 
   // The --thinlto-cache-dir option specifies the path to a directory in which
   // to cache native object files for ThinLTO incremental builds. If a path was
   // specified, configure LTO to use it as the cache directory.
-  NativeObjectCache cache;
+  FileCache cache;
   if (!config->thinLTOCacheDir.empty())
     cache =
         check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
                          [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
                            files[task] = std::move(mb);
                          }));
 
   if (!bitcodeFiles.empty())
     checkError(ltoObj->run(
         [&](size_t task) {
-          return std::make_unique<NativeObjectStream>(
+          return std::make_unique<CachedFileStream>(
               std::make_unique<raw_svector_ostream>(buf[task]));
         },
         cache));
 
   // Emit empty index files for non-indexed files but not in single-module mode.
   if (config->thinLTOModulesToCompile.empty()) {
     for (StringRef s : thinIndices) {
       std::string path = getThinLTOOutputFile(s);

lld/MachO/LTO.cpp

 std::vector<ObjFile *> BitcodeCompiler::compile() {
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
   files.resize(maxTasks);
 
   // The -cache_path_lto option specifies the path to a directory in which
   // to cache native object files for ThinLTO incremental builds. If a path was
   // specified, configure LTO to use it as the cache directory.
-  NativeObjectCache cache;
+  FileCache cache;
   if (!config->thinLTOCacheDir.empty())
     cache =
         check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
                          [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
                            files[task] = std::move(mb);
                          }));
 
   checkError(ltoObj->run(
       [&](size_t task) {
-        return std::make_unique<NativeObjectStream>(
+        return std::make_unique<CachedFileStream>(
             std::make_unique<raw_svector_ostream>(buf[task]));
       },
       cache));
 
   if (!config->thinLTOCacheDir.empty())
     pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
 
   if (config->saveTemps) {

lld/wasm/LTO.cpp

 std::vector<StringRef> BitcodeCompiler::compile() {
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
   files.resize(maxTasks);
 
   // The --thinlto-cache-dir option specifies the path to a directory in which
   // to cache native object files for ThinLTO incremental builds. If a path was
   // specified, configure LTO to use it as the cache directory.
-  NativeObjectCache cache;
+  FileCache cache;
   if (!config->thinLTOCacheDir.empty())
     cache =
         check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
                          [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
                            files[task] = std::move(mb);
                          }));
 
   checkError(ltoObj->run(
       [&](size_t task) {
-        return std::make_unique<NativeObjectStream>(
+        return std::make_unique<CachedFileStream>(
             std::make_unique<raw_svector_ostream>(buf[task]));
       },
       cache));
 
   if (!config->thinLTOCacheDir.empty())
     pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
 
   std::vector<StringRef> ret;

llvm/include/llvm/LTO/LTO.h

 };
 
 /// A ThinBackend defines what happens after the thin-link phase during ThinLTO.
 /// The details of this type definition aren't important; clients can only
 /// create a ThinBackend using one of the create*ThinBackend() functions below.
 using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>(
     const Config &C, ModuleSummaryIndex &CombinedIndex,
     StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-    AddStreamFn AddStream, NativeObjectCache Cache)>;
+    AddStreamFn AddStream, FileCache Cache)>;
 
 /// This ThinBackend runs the individual backend jobs in-process.
 /// The default value means to use one job per hardware core (not hyper-thread).
 ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism);
 
 /// This ThinBackend writes individual module indexes to files, instead of
 /// running the individual backend jobs. This backend is for distributed builds
 /// where separate processes will invoke the real backends.
public:
   /// Runs the LTO pipeline. This function calls the supplied AddStream
   /// function to add native object files to the link.
   ///
   /// The Cache parameter is optional. If supplied, it will be used to cache
   /// native object files and add them to the link.
   ///
   /// The client will receive at most one callback (via either AddStream or
   /// Cache) for each task identifier.
-  Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);
+  Error run(AddStreamFn AddStream, FileCache Cache = nullptr);
 
   /// Static method that returns a list of libcall symbols that can be generated
   /// by LTO but might not be visible from bitcode symbol table.
   static ArrayRef<const char*> getRuntimeLibcallSymbols();
 
 private:
   Config Conf;
 
addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
                 const SymbolResolution *&ResI, const SymbolResolution *ResE);
   Error linkRegularLTO(RegularLTOState::AddedModule Mod,
                        bool LivenessFromIndex);
 
   Error addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
                    const SymbolResolution *&ResI, const SymbolResolution *ResE);
 
   Error runRegularLTO(AddStreamFn AddStream);
-  Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
+  Error runThinLTO(AddStreamFn AddStream, FileCache Cache,
                    const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols);
 
   Error checkPartiallySplit();
 
   mutable bool CalledGetMaxTasks = false;
 
   // Use Optional to distinguish false from not yet initialized.
   Optional<bool> EnableSplitLTOUnit;

llvm/include/llvm/Support/Caching.h

-//===- Caching.h - LLVM File Cache Handling Configuration -------*- C++ -*-===//
+//===- Caching.h - LLVM Local File Cache ------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines the localCache function, which allows clients to add a
-// filesystem cache. This is used by ThinLTO.
+// This file defines the CachedFileStream and the localCache function, which
+// simplifies caching files on the local filesystem in a directory whose
+// contents are managed by a CachePruningPolicy.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_SUPPORT_CACHING_H
 #define LLVM_SUPPORT_CACHING_H
 
 #include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/thread.h"
 
 namespace llvm {
 
-/// This class wraps an output stream for a native object. Most clients should
-/// just be able to return an instance of this base class from the stream
-/// callback, but if a client needs to perform some action after the stream is
-/// written to, that can be done by deriving from this class and overriding the
-/// destructor.
-class NativeObjectStream {
+class MemoryBuffer;
+
+/// This class wraps an output stream for a file. Most clients should just be
+/// able to return an instance of this base class from the stream callback, but
+/// if a client needs to perform some action after the stream is written to,
+/// that can be done by deriving from this class and overriding the destructor.
+class CachedFileStream {
 public:
-  NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS)
-      : OS(std::move(OS)) {}
+  CachedFileStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}
   std::unique_ptr<raw_pwrite_stream> OS;
-  virtual ~NativeObjectStream() = default;
+  virtual ~CachedFileStream() = default;
 };
 
-/// This type defines the callback to add a native object that is generated on
-/// the fly.
+/// This type defines the callback to add a file that is generated on the fly.
 ///
 /// Stream callbacks must be thread safe.
 using AddStreamFn =
-    std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>;
+    std::function<Expected<std::unique_ptr<CachedFileStream>>(unsigned Task)>;
 
-/// This is the type of a native object cache. To request an item from the
-/// cache, pass a unique string as the Key. For hits, the cached file will be
-/// added to the link and this function will return AddStreamFn(). For misses,
-/// the cache will return a stream callback which must be called at most once to
-/// produce content for the stream. The native object stream produced by the
-/// stream callback will add the file to the link after the stream is written
-/// to.
+/// This is the type of a file cache. To request an item from the cache, pass a
+/// unique string as the Key. For hits, the cached file will be added to the
+/// link and this function will return AddStreamFn(). For misses, the cache will
+/// return a stream callback which must be called at most once to produce
+/// content for the stream. The file stream produced by the stream callback will
+/// add the file to the link after the stream is written to.
 ///
 /// Clients generally look like this:
 ///
 /// if (AddStreamFn AddStream = Cache(Task, Key))
 ///   ProduceContent(AddStream);
-using NativeObjectCache =
-    std::function<AddStreamFn(unsigned Task, StringRef Key)>;
+using FileCache =
+    std::function<Expected<AddStreamFn>(unsigned Task, StringRef Key)>;
 
-/// This type defines the callback to add a pre-existing native object file
-/// (e.g. in a cache).
+/// This type defines the callback to add a pre-existing file (e.g. in a cache).
 ///
 /// Buffer callbacks must be thread safe.
 using AddBufferFn =
     std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)>;
 
 /// Create a local file system cache which uses the given cache name, temporary
 /// file prefix, cache directory and file callback. This function also creates
 /// the cache directory if it does not already exist. The cache name appears in
 /// error messages for errors during caching. The temporary file prefix is used
 /// in the temporary file naming scheme used when writing files atomically.
-Expected<NativeObjectCache> localCache(Twine CacheNameRef,
-                                       Twine TempFilePrefixRef,
-                                       Twine CacheDirectoryPathRef,
-                                       AddBufferFn AddBuffer);
+Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
 } // namespace llvm
 
 #endif

llvm/lib/LTO/LTO.cpp

for (auto &S : P.second.SummaryList) {
         return make_error<StringError>(
             "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
             inconvertibleErrorCode());
     }
   }
   return Error::success();
 }
 
-Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
+Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
   // Compute "dead" symbols, we don't want to import/export these!
   DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
   DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
   for (auto &Res : GlobalResolutions) {
     // Normally resolution have IR name of symbol. We can do nothing here
     // otherwise. See comments in GlobalResolution struct for more details.
     if (Res.second.IRName.empty())
       continue;
public:
   virtual Error wait() = 0;
   virtual unsigned getThreadCount() = 0;
 };
 
 namespace {
 class InProcessThinBackend : public ThinBackendProc {
   ThreadPool BackendThreadPool;
   AddStreamFn AddStream;
-  NativeObjectCache Cache;
+  FileCache Cache;
   std::set<GlobalValue::GUID> CfiFunctionDefs;
   std::set<GlobalValue::GUID> CfiFunctionDecls;
 
   Optional<Error> Err;
   std::mutex ErrMu;
 
 public:
   InProcessThinBackend(
       const Config &Conf, ModuleSummaryIndex &CombinedIndex,
       ThreadPoolStrategy ThinLTOParallelism,
       const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-      AddStreamFn AddStream, NativeObjectCache Cache)
+      AddStreamFn AddStream, FileCache Cache)
       : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
         BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
         Cache(std::move(Cache)) {
     for (auto &Name : CombinedIndex.cfiFunctionDefs())
       CfiFunctionDefs.insert(
           GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
     for (auto &Name : CombinedIndex.cfiFunctionDecls())
       CfiFunctionDecls.insert(
           GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
   }
 
   Error runThinLTOBackendThread(
-      AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
-      BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
+      AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
+      ModuleSummaryIndex &CombinedIndex,
       const FunctionImporter::ImportMapTy &ImportList,
       const FunctionImporter::ExportSetTy &ExportList,
       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
       const GVSummaryMapTy &DefinedGlobals,
       MapVector<StringRef, BitcodeModule> &ModuleMap) {
     auto RunThinBackend = [&](AddStreamFn AddStream) {
       LTOLLVMContext BackendContext(Conf);
       Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
       // no module hash.
       return RunThinBackend(AddStream);
 
     SmallString<40> Key;
     // The module may be cached, this helps handling it.
     computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
                        ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
                        CfiFunctionDecls);
-    if (AddStreamFn CacheAddStream = Cache(Task, Key))
+    Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key);
+    if (Error Err = CacheAddStreamOrErr.takeError())
+      return Err;
+    AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
+    if (CacheAddStream)
       return RunThinBackend(CacheAddStream);
 
     return Error::success();
   }
 
   Error start(
       unsigned Task, BitcodeModule BM,
       const FunctionImporter::ImportMapTy &ImportList,
unsigned getThreadCount() override {
     return BackendThreadPool.getThreadCount();
   }
 };
 } // end anonymous namespace
 
 ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism) {
   return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
              const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-             AddStreamFn AddStream, NativeObjectCache Cache) {
+             AddStreamFn AddStream, FileCache Cache) {
     return std::make_unique<InProcessThinBackend>(
         Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream,
         Cache);
   };
 }
 
 // Given the original \p Path to an output file, replace any path
 // prefix matching \p OldPrefix with \p NewPrefix. Also, create the

 };
 } // end anonymous namespace
 
 ThinBackend lto::createWriteIndexesThinBackend(
     std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
     raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
   return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
              const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-             AddStreamFn AddStream, NativeObjectCache Cache) {
+             AddStreamFn AddStream, FileCache Cache) {
     return std::make_unique<WriteIndexesThinBackend>(
         Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
         ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
   };
 }
 
-Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
+Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
                       const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
   timeTraceProfilerBegin("ThinLink", StringRef(""));
   auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
     if (llvm::timeTraceProfilerEnabled())
       llvm::timeTraceProfilerEnd();
   });
   if (ThinLTO.ModuleMap.empty())
     return Error::success();

llvm/lib/LTO/LTOBackend.cpp

static void codegen(const Config &Conf, TargetMachine *TM,
   if (!DwoFile.empty()) {
     std::error_code EC;
     DwoOut = std::make_unique<ToolOutputFile>(DwoFile, EC, sys::fs::OF_None);
     if (EC)
       report_fatal_error(Twine("Failed to open ") + DwoFile + ": " +
                          EC.message());
   }
 
-  auto Stream = AddStream(Task);
+  Expected<std::unique_ptr<CachedFileStream>> StreamOrErr = AddStream(Task);
+  if (Error Err = StreamOrErr.takeError())
+    report_fatal_error(std::move(Err));
+  std::unique_ptr<CachedFileStream> &Stream = *StreamOrErr;
   legacy::PassManager CodeGenPasses;
   CodeGenPasses.add(
       createImmutableModuleSummaryIndexWrapperPass(&CombinedIndex));
   if (Conf.PreCodeGenPassesHook)
     Conf.PreCodeGenPassesHook(CodeGenPasses);
   if (TM->addPassesToEmitFile(CodeGenPasses, *Stream->OS,
                               DwoOut ? &DwoOut->os() : nullptr,
                               Conf.CGFileType))

llvm/lib/LTO/LTOCodeGenerator.cpp

bool LTOCodeGenerator::writeMergedModules(StringRef Path) {
   Out.keep();
   return true;
 }
 
 bool LTOCodeGenerator::compileOptimizedToFile(const char **Name) {
   // make unique temp output file to put generated code
   SmallString<128> Filename;
 
-  auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
     StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o");
 
     int FD;
     std::error_code EC =
         sys::fs::createTemporaryFile("lto-llvm", Extension, FD, Filename);
     if (EC)
       emitError(EC.message());
 
-    return std::make_unique<NativeObjectStream>(
+    return std::make_unique<CachedFileStream>(
         std::make_unique<llvm::raw_fd_ostream>(FD, true));
   };
 
   bool genResult = compileOptimized(AddStream, 1);
 
   if (!genResult) {
     sys::fs::remove(Twine(Filename));
     return false;

llvm/lib/Support/Caching.cpp

-//===-Caching.cpp - LLVM File Cache Handling ------------------------------===//
+//===-Caching.cpp - LLVM Local File Cache ---------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the Caching used by ThinLTO.
+// This file implements the localCache function, which simplifies creating,
+// adding to, and querying a local file system cache. localCache takes care of
+// periodically pruning older files from the cache using a CachePruningPolicy.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/Caching.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/Process.h"
-#include "llvm/Support/raw_ostream.h"
 
 #if !defined(_MSC_VER) && !defined(__MINGW32__)
 #include <unistd.h>
 #else
 #include <io.h>
 #endif
 
 using namespace llvm;
 
-Expected<NativeObjectCache> llvm::localCache(Twine CacheNameRef,
-                                             Twine TempFilePrefixRef,
-                                             Twine CacheDirectoryPathRef,
-                                             AddBufferFn AddBuffer) {
+Expected<FileCache> llvm::localCache(Twine CacheNameRef,
   if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPathRef))
     return errorCodeToError(EC);
 
   // Create local copies which are safely captured-by-copy in lambdas
   SmallString<64> CacheName, TempFilePrefix, CacheDirectoryPath;
   CacheNameRef.toVector(CacheName);
   TempFilePrefixRef.toVector(TempFilePrefix);
   CacheDirectoryPathRef.toVector(CacheDirectoryPath);
 
-  return [=](unsigned Task, StringRef Key) -> AddStreamFn {
+  return [=](unsigned Task, StringRef Key) -> Expected<AddStreamFn> {
     // This choice of file name allows the cache to be pruned (see pruneCache()
     // in include/llvm/Support/CachePruning.h).
     SmallString<64> EntryPath;
     sys::path::append(EntryPath, CacheDirectoryPath, "llvmcache-" + Key);
     // First, see if we have a cache hit.
     SmallString<64> ResultPath;
     Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
         Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
return [=](unsigned Task, StringRef Key) -> Expected<AddStreamFn> {
 
     // On Windows we can fail to open a cache file with a permission denied
     // error. This generally means that another process has requested to delete
     // the file while it is still open, but it could also mean that another
     // process has opened the file without the sharing permissions we need.
     // Since the file is probably being deleted we handle it in the same way as
     // if the file did not exist at all.
     if (EC != errc::no_such_file_or_directory && EC != errc::permission_denied)
-      report_fatal_error(Twine("Failed to open cache file ") + EntryPath +
-                         ": " + EC.message() + "\n");
+      return createStringError(EC, Twine("Failed to open cache file ") +
+                                       EntryPath + ": " + EC.message() + "\n");
 
-    // This native object stream is responsible for commiting the resulting
-    // file to the cache and calling AddBuffer to add it to the link.
-    struct CacheStream : NativeObjectStream {
+    // This file stream is responsible for commiting the resulting file to the
+    // cache and calling AddBuffer to add it to the link.
+    struct CacheStream : CachedFileStream {
       AddBufferFn AddBuffer;
       sys::fs::TempFile TempFile;
       std::string EntryPath;
       unsigned Task;
 
       CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
                   sys::fs::TempFile TempFile, std::string EntryPath,
                   unsigned Task)
-          : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
+          : CachedFileStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
             TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
             Task(Task) {}
 
       ~CacheStream() {
+        // TODO: Manually commit rather than using non-trivial destructor,
+        // allowing to replace report_fatal_errors with a return Error.
+
         // Make sure the stream is closed before committing it.
         OS.reset();
 
         // Open the file first to avoid racing with a cache pruner.
         ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
             MemoryBuffer::getOpenFile(
                 sys::fs::convertFDToNativeFile(TempFile.FD), TempFile.TmpName,
                 /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
struct CacheStream : CachedFileStream {
           report_fatal_error(Twine("Failed to rename temporary file ") +
                              TempFile.TmpName + " to " + EntryPath + ": " +
                              toString(std::move(E)) + "\n");
 
         AddBuffer(Task, std::move(*MBOrErr));
       }
     };
 
-    return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+    return [=](size_t Task) -> Expected<std::unique_ptr<CachedFileStream>> {
       // Write to a temporary to avoid race condition
       SmallString<64> TempFilenameModel;
       sys::path::append(TempFilenameModel, CacheDirectoryPath,
                         TempFilePrefix + "-%%%%%%.tmp.o");
       Expected<sys::fs::TempFile> Temp = sys::fs::TempFile::create(
           TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write);
-      if (!Temp) {
-        errs() << "Error: " << toString(Temp.takeError()) << "\n";
-        report_fatal_error(CacheName + ": Can't get a temporary file");
-      }
+      if (!Temp)
+        return createStringError(errc::io_error,
+                                 toString(Temp.takeError()) + ": " + CacheName +
+                                     ": Can't get a temporary file");
 
       // This CacheStream will move the temporary file into the cache when done.
       return std::make_unique<CacheStream>(
           std::make_unique<raw_fd_ostream>(Temp->FD, /* ShouldClose */ false),
           AddBuffer, std::move(*Temp), std::string(EntryPath.str()), Task);
     };
   };
 }

llvm/tools/gold/gold-plugin.cpp

else if (options::TheOutputType == options::OT_SAVE_TEMPS)
     Filename = output_name + ".lto.o";
   else if (options::TheOutputType == options::OT_ASM_ONLY)
     Filename = output_name;
   bool SaveTemps = !Filename.empty();
 
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector<std::pair<SmallString<128>, bool>> Files(MaxTasks);
 
-  auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
     Files[Task].second = !SaveTemps;
     int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
                                Files[Task].first, Task);
-    return std::make_unique<NativeObjectStream>(
+    return std::make_unique<CachedFileStream>(
         std::make_unique<llvm::raw_fd_ostream>(FD, true));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
     *AddStream(Task)->OS << MB->getBuffer();
   };
 
-  NativeObjectCache Cache;
+  FileCache Cache;
   if (!options::cache_dir.empty())
     Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
   // Write empty output files that may be expected by the distributed build
   // system.
   if (options::thinlto_index_only)

llvm/tools/llvm-lto/llvm-lto.cpp

if (SaveModuleFile) {
       std::string ModuleFilename = OutputFilename;
       ModuleFilename += ".merged.bc";
       std::string ErrMsg;
 
       if (!CodeGen.writeMergedModules(ModuleFilename))
         error("writing merged module failed.");
     }
 
-    auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+    auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
       std::string PartFilename = OutputFilename;
       if (Parallelism != 1)
         PartFilename += "." + utostr(Task);
 
       std::error_code EC;
       auto S =
           std::make_unique<raw_fd_ostream>(PartFilename, EC, sys::fs::OF_None);
       if (EC)
         error("error opening the file '" + PartFilename + "': " + EC.message());
-      return std::make_unique<NativeObjectStream>(std::move(S));
+      return std::make_unique<CachedFileStream>(std::move(S));
     };
 
     if (!CodeGen.compileOptimized(AddStream, Parallelism))
       // Diagnostic messages should have been printed by the handler.
       error("error compiling the code");
 
   } else {
     if (Parallelism != 1)

llvm/tools/llvm-lto2/llvm-lto2.cpp

if (!CommandLineResolutions.empty()) {
     for (auto UnusedRes : CommandLineResolutions)
       llvm::errs() << argv[0] << ": unused symbol resolution for "
                    << UnusedRes.first.first << ',' << UnusedRes.first.second
                    << '\n';
   }
   if (HasErrors)
     return 1;
 
-  auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
     std::string Path = OutputFilename + "." + utostr(Task);
 
     std::error_code EC;
     auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None);
     check(EC, Path);
-    return std::make_unique<NativeObjectStream>(std::move(S));
+    return std::make_unique<CachedFileStream>(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
     *AddStream(Task)->OS << MB->getBuffer();
   };
 
-  NativeObjectCache Cache;
+  FileCache Cache;
   if (!CacheDir.empty())
     Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
                   "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
 }