[Support] Add overload writeFileAtomically(std::function Writer)

[jkorous]

Summary

I noticed there are couple other implementations of atomic file write and replaced these with shared implementation.
I have to admit my enthusiasm for factoring this out waned a bit after I added proper error handling in lldb-platform and ThinLTOCodeGenerator as it doesn't look all that simpler than current code.
(Maybe I'm holding it wrong?) Open to suggestions.

[JDevlieghere]

Thanks Jan, I think this is a great improvement. IMHO the fact that we can share code outweighs the slightly increased complexity.

llvm/include/llvm/Support/FileUtilities.h

107 ↗

(On Diff #219637)

I wonder if the temp path should be configurable or at least make it a default argument. Removing that would definitely make the call sites a little simpler. At least for LLDB I don't think the temp path matters.

[gribozavr]

lldb/tools/lldb-server/lldb-platform.cpp

24 ↗

(On Diff #219637)

Move the new header here, to group with the rest?

llvm/lib/Support/FileUtilities.cpp

332 ↗

(On Diff #219637)

Please add a newline.

[jkorous]

llvm/include/llvm/Support/FileUtilities.h

107 ↗

(On Diff #219637)

Of course, good point! I'll create a separate patch.

cfe/trunk/lib/Frontend/ASTUnit.cpp

 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/DJB.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <atomic>
 #include <cassert>
 #include <cstdint>
bool ASTUnit::Save(StringRef File) {
   if (HadModuleLoaderFatalFailure)
     return true;
 
   // Write to a temporary file and later rename it to the actual file, to avoid
   // possible race conditions.
   SmallString<128> TempPath;
   TempPath = File;
   TempPath += "-%%%%%%%%";
-  int fd;
-  if (llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath))
-    return true;
-
   // FIXME: Can we somehow regenerate the stat cache here, or do we need to
   // unconditionally create a stat cache when we parse the file?
-  llvm::raw_fd_ostream Out(fd, /*shouldClose=*/true);
-
-  serialize(Out);
-  Out.close();
-  if (Out.has_error()) {
-    Out.clear_error();
-    return true;
-  }
 
-  if (llvm::sys::fs::rename(TempPath, File)) {
-    llvm::sys::fs::remove(TempPath);
+  if (llvm::Error Err = llvm::writeFileAtomically(
+          TempPath, File, [this](llvm::raw_ostream &Out) {
+            return serialize(Out) ? llvm::make_error<llvm::StringError>(
+                                        "ASTUnit serialization failed",
+                                        llvm::inconvertibleErrorCode())
+                                  : llvm::Error::success();
+          })) {
+    consumeError(std::move(Err));
     return true;
   }
-
   return false;
 }
 
 static bool serializeUnit(ASTWriter &Writer,
                           SmallVectorImpl<char> &Buffer,
                           Sema &S,
                           bool hasErrors,
                           raw_ostream &OS) {

cfe/trunk/lib/Serialization/GlobalModuleIndex.cpp

 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the GlobalModuleIndex class.
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "ASTReaderInternals.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/GlobalModuleIndex.h"
 #include "clang/Serialization/Module.h"
 #include "clang/Serialization/PCHContainerOperations.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Support/DJB.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/LockFileManager.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/OnDiskHashTable.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TimeProfiler.h"
 #include <cstdio>
 using namespace clang;
 using namespace serialization;
GlobalModuleIndex::writeIndex(FileManager &FileMgr,
   SmallVector<char, 16> OutputBuffer;
   {
     llvm::BitstreamWriter OutputStream(OutputBuffer);
     if (Builder.writeIndex(OutputStream))
       return llvm::createStringError(std::errc::io_error,
                                      "failed writing index");
   }
 
-  // Write the global index file to a temporary file.
-  llvm::SmallString<128> IndexTmpPath;
-  int TmpFD;
-  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
-                                      IndexTmpPath))
-    return llvm::createStringError(std::errc::io_error,
-                                   "failed creating unique file");
-
-  // Open the temporary global index file for output.
-  llvm::raw_fd_ostream Out(TmpFD, true);
-  if (Out.has_error())
-    return llvm::createStringError(Out.error(), "failed outputting to stream");
-
-  // Write the index.
-  Out.write(OutputBuffer.data(), OutputBuffer.size());
-  Out.close();
-  if (Out.has_error())
-    return llvm::createStringError(Out.error(), "failed writing to stream");
-
-  // Remove the old index file. It isn't relevant any more.
-  llvm::sys::fs::remove(IndexPath);
-
-  // Rename the newly-written index file to the proper name.
-  if (std::error_code Err = llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
-    // Remove the file on failure, don't check whether removal succeeded.
-    llvm::sys::fs::remove(IndexTmpPath);
-    return llvm::createStringError(Err, "failed renaming file \"%s\" to \"%s\"",
-                                   IndexTmpPath.c_str(), IndexPath.c_str());
-  }
-
-  return llvm::Error::success();
+  return llvm::writeFileAtomically(
+      (IndexPath + "-%%%%%%%%").str(), IndexPath,
+      llvm::StringRef(OutputBuffer.data(), OutputBuffer.size()));
 }
 
 namespace {
   class GlobalIndexIdentifierIterator : public IdentifierIterator {
     /// The current position within the identifier lookup table.
     IdentifierIndexTable::key_iterator Current;
 
     /// The end position within the identifier lookup table.

clang-tools-extra/trunk/clangd/index/BackgroundIndexStorage.cpp

 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include <functional>
 
 namespace clang {
 namespace clangd {
 namespace {
 
 std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
                                      llvm::StringRef FilePath) {
   llvm::SmallString<128> ShardRootSS(ShardRoot);
   llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) +
                                            "." + llvm::toHex(digest(FilePath)) +
                                            ".idx");
   return ShardRootSS.str();
 }
 
-llvm::Error
-writeAtomically(llvm::StringRef OutPath,
-                llvm::function_ref<void(llvm::raw_ostream &)> Writer) {
-  // Write to a temporary file first.
-  llvm::SmallString<128> TempPath;
-  int FD;
-  auto EC =
-      llvm::sys::fs::createUniqueFile(OutPath + ".tmp.%%%%%%%%", FD, TempPath);
-  if (EC)
-    return llvm::errorCodeToError(EC);
-  // Make sure temp file is destroyed on failure.
-  auto RemoveOnFail =
-      llvm::make_scope_exit([TempPath] { llvm::sys::fs::remove(TempPath); });
-  llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
-  Writer(OS);
-  OS.close();
-  if (OS.has_error())
-    return llvm::errorCodeToError(OS.error());
-  // Then move to real location.
-  EC = llvm::sys::fs::rename(TempPath, OutPath);
-  if (EC)
-    return llvm::errorCodeToError(EC);
-  // If everything went well, we already moved the file to another name. So
-  // don't delete the file, as the name might be taken by another file.
-  RemoveOnFail.release();
-  return llvm::ErrorSuccess();
-}
-
 // Uses disk as a storage for index shards. Creates a directory called
 // ".clangd/index/" under the path provided during construction.
 class DiskBackedIndexStorage : public BackgroundIndexStorage {
   std::string DiskShardRoot;
 
 public:
   // Sets DiskShardRoot to (Directory + ".clangd/index/") which is the base
   // directory for all shard files.
loadShard(llvm::StringRef ShardIdentifier) const override {
     else
       elog("Error while reading shard {0}: {1}", ShardIdentifier,
            I.takeError());
     return nullptr;
   }
 
   llvm::Error storeShard(llvm::StringRef ShardIdentifier,
                          IndexFileOut Shard) const override {
-    return writeAtomically(
-        getShardPathFromFilePath(DiskShardRoot, ShardIdentifier),
-        [&Shard](llvm::raw_ostream &OS) { OS << Shard; });
+    auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
+    return llvm::writeFileAtomically(ShardPath + ".tmp.%%%%%%%%", ShardPath,
+                                     [&Shard](llvm::raw_ostream &OS) {
+                                       OS << Shard;
+                                       return llvm::Error::success();
+                                     });
   }
 };
 
 // Doesn't persist index shards anywhere (used when the CDB dir is unknown).
 // We could consider indexing into ~/.clangd/ or so instead.
 class NullStorage : public BackgroundIndexStorage {
 public:
   std::unique_ptr<IndexFileIn>

lldb/trunk/tools/lldb-server/lldb-platform.cpp

 #include <string.h>
 #if !defined(_WIN32)
 #include <sys/wait.h>
 #endif
 #include <fstream>
 
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include "Acceptor.h"
 #include "LLDBServerUtilities.h"
 #include "Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h"
 #include "Plugins/Process/gdb-remote/ProcessGDBRemoteLog.h"
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/HostGetOpt.h"
 #include "lldb/Host/OptionParser.h"
static Status save_socket_id_to_file(const std::string &socket_id,
   FileSpec temp_file_spec(file_spec.GetDirectory().AsCString());
   Status error(llvm::sys::fs::create_directory(temp_file_spec.GetPath()));
   if (error.Fail())
     return Status("Failed to create directory %s: %s",
                   temp_file_spec.GetCString(), error.AsCString());
 
   llvm::SmallString<64> temp_file_path;
   temp_file_spec.AppendPathComponent("port-file.%%%%%%");
-  int FD;
-  auto err_code = llvm::sys::fs::createUniqueFile(temp_file_spec.GetPath(), FD,
-                                                  temp_file_path);
-  if (err_code)
-    return Status("Failed to create temp file: %s", err_code.message().c_str());
-
-  llvm::FileRemover tmp_file_remover(temp_file_path);
-
-  {
-    llvm::raw_fd_ostream temp_file(FD, true);
-    temp_file << socket_id;
-    temp_file.close();
-    if (temp_file.has_error())
-      return Status("Failed to write to port file.");
-  }
-
-  err_code = llvm::sys::fs::rename(temp_file_path, file_spec.GetPath());
-  if (err_code)
-    return Status("Failed to rename file %s to %s: %s", temp_file_path.c_str(),
-                  file_spec.GetPath().c_str(), err_code.message().c_str());
 
-  tmp_file_remover.releaseFile();
-  return Status();
+  Status status;
+  if (auto Err =
+          handleErrors(llvm::writeFileAtomically(
+                           temp_file_path, temp_file_spec.GetPath(), socket_id),
+                       [&status, &temp_file_path,
+                        &file_spec](const AtomicFileWriteError &E) {
+                         std::string ErrorMsgBuffer;
+                         llvm::raw_string_ostream S(ErrorMsgBuffer);
+                         E.log(S);
+
+                         switch (E.Error) {
+                         case atomic_write_error::failed_to_create_uniq_file:
+                           status = Status("Failed to create temp file: %s",
+                                           ErrorMsgBuffer.c_str());
+                         case atomic_write_error::output_stream_error:
+                           status = Status("Failed to write to port file.");
+                         case atomic_write_error::failed_to_rename_temp_file:
+                           status = Status("Failed to rename file %s to %s: %s",
+                                           ErrorMsgBuffer.c_str(),
+                                           file_spec.GetPath().c_str(),
+                                           ErrorMsgBuffer.c_str());
+                         }
+                       })) {
+    return Status("Failed to atomically write file %s",
+                  file_spec.GetPath().c_str());
+  }
+  return status;
 }
 
 // main
 int main_platform(int argc, char *argv[]) {
   const char *progname = argv[0];
   const char *subcommand = argv[1];
   argc--;
   argv++;

llvm/trunk/include/llvm/Support/FileUtilities.h

 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_SUPPORT_FILEUTILITIES_H
 #define LLVM_SUPPORT_FILEUTILITIES_H
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
 namespace llvm {
 
   /// DiffFilesWithTolerance - Compare the two files specified, returning 0 if
   /// the files match, 1 if they are different, and 2 if there is a file error.
   /// This function allows you to specify an absolute and relative FP error that
void setFile(const Twine& filename, bool deleteIt = true) {
       DeleteIt = deleteIt;
     }
 
     /// releaseFile - Take ownership of the file away from the FileRemover so it
     /// will not be removed when the object is destroyed.
     void releaseFile() { DeleteIt = false; }
   };
 
+  enum class atomic_write_error {
+    failed_to_create_uniq_file = 0,
+    output_stream_error,
+    failed_to_rename_temp_file
+  };
+
+  class AtomicFileWriteError : public llvm::ErrorInfo<AtomicFileWriteError> {
+  public:
+    AtomicFileWriteError(atomic_write_error Error) : Error(Error) {}
+
+    void log(raw_ostream &OS) const override;
+
+    const atomic_write_error Error;
+    static char ID;
+
+  private:
+    // Users are not expected to use error_code.
+    std::error_code convertToErrorCode() const override {
+      return llvm::inconvertibleErrorCode();
+    }
+  };
+
+  // atomic_write_error + whatever the Writer can return
+
   /// Creates a unique file with name according to the given \p TempPathModel,
   /// writes content of \p Buffer to the file and renames it to \p FinalPath.
+  ///
+  /// \returns \c AtomicFileWriteError in case of error.
   llvm::Error writeFileAtomically(StringRef TempPathModel, StringRef FinalPath,
                                   StringRef Buffer);
+
+  llvm::Error
+  writeFileAtomically(StringRef TempPathModel, StringRef FinalPath,
+                      std::function<llvm::Error(llvm::raw_ostream &)> Writer);
 } // End llvm namespace
 
 #endif

llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp

 #include "llvm/IRReader/IRReader.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/LTO/SummaryBasedOptimizations.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SHA1.h"
 #include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/VCSRevision.h"
public:
   // Cache the Produced object file
   void write(const MemoryBuffer &OutputBuffer) {
     if (EntryPath.empty())
       return;
 
     // Write to a temporary to avoid race condition
     SmallString<128> TempFilename;
     SmallString<128> CachePath(EntryPath);
-    int TempFD;
     llvm::sys::path::remove_filename(CachePath);
     sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
-    std::error_code EC =
-      sys::fs::createUniqueFile(TempFilename, TempFD, TempFilename);
-    if (EC) {
-      errs() << "Error: " << EC.message() << "\n";
+
+    if (auto Err = handleErrors(
+            llvm::writeFileAtomically(TempFilename, EntryPath,
+                                      OutputBuffer.getBuffer()),
+            [](const llvm::AtomicFileWriteError &E) {
+              std::string ErrorMsgBuffer;
+              llvm::raw_string_ostream S(ErrorMsgBuffer);
+              E.log(S);
+
+              if (E.Error ==
+                  llvm::atomic_write_error::failed_to_create_uniq_file) {
+                errs() << "Error: " << ErrorMsgBuffer << "\n";
                 report_fatal_error("ThinLTO: Can't get a temporary file");
               }
-    {
-      raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
-      OS << OutputBuffer.getBuffer();
+            })) {
+      // FIXME
+      consumeError(std::move(Err));
     }
-    // Rename temp file to final destination; rename is atomic
-    EC = sys::fs::rename(TempFilename, EntryPath);
-    if (EC)
-      sys::fs::remove(TempFilename);
   }
 };
 
 static std::unique_ptr<MemoryBuffer>
 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
                      StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
                      const FunctionImporter::ImportMapTy &ImportList,
                      const FunctionImporter::ExportSetTy &ExportList,

llvm/trunk/lib/Support/FileUtilities.cpp

 //===- Support/FileUtilities.cpp - File System Utilities ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a family of utility functions which are useful for doing
 // various things with files.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/FileUtilities.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cctype>
 #include <cmath>
 #include <cstdint>
 #include <cstdlib>
if (!CompareFailed && (!F1AtEnd || !F2AtEnd)) {
     // If we found the end, we succeeded.
     if (F1P < File1End || F2P < File2End)
       CompareFailed = true;
   }
 
   return CompareFailed;
 }
 
-Error llvm::writeFileAtomically(StringRef TempPathModel, StringRef FinalPath,
-                                StringRef Buffer) {
+void llvm::AtomicFileWriteError::log(raw_ostream &OS) const {
+  OS << "atomic_write_error: ";
+  switch (Error) {
+  case atomic_write_error::failed_to_create_uniq_file:
+    OS << "failed_to_create_uniq_file";
+    return;
+  case atomic_write_error::output_stream_error:
+    OS << "output_stream_error";
+    return;
+  case atomic_write_error::failed_to_rename_temp_file:
+    OS << "failed_to_rename_temp_file";
+    return;
+  }
+  llvm_unreachable("unknown atomic_write_error value in "
+                   "failed_to_rename_temp_file::log()");
+}
+
+llvm::Error llvm::writeFileAtomically(StringRef TempPathModel,
+                                      StringRef FinalPath, StringRef Buffer) {
+  return writeFileAtomically(TempPathModel, FinalPath,
+                             [&Buffer](llvm::raw_ostream &OS) {
+                               OS.write(Buffer.data(), Buffer.size());
+                               return llvm::Error::success();
+                             });
+}
+
+llvm::Error llvm::writeFileAtomically(
+    StringRef TempPathModel, StringRef FinalPath,
+    std::function<llvm::Error(llvm::raw_ostream &)> Writer) {
   SmallString<128> GeneratedUniqPath;
   int TempFD;
-  if (const std::error_code Error = sys::fs::createUniqueFile(
-          TempPathModel.str(), TempFD, GeneratedUniqPath)) {
-    return createStringError(
-        Error, "failed to create temporary file with model \"%s\"",
-        TempPathModel.str().c_str());
+  if (sys::fs::createUniqueFile(TempPathModel.str(), TempFD,
+                                GeneratedUniqPath)) {
+    return llvm::make_error<AtomicFileWriteError>(
+        atomic_write_error::failed_to_create_uniq_file);
   }
+  llvm::FileRemover RemoveTmpFileOnFail(GeneratedUniqPath);
 
   raw_fd_ostream OS(TempFD, /*shouldClose=*/true);
-  OS.write(Buffer.data(), Buffer.size());
-  OS.close();
-  TempFD = -1;
+  if (llvm::Error Err = Writer(OS)) {
+    return Err;
+  }
 
+  OS.close();
   if (OS.has_error()) {
-    const std::error_code Error = OS.error();
     OS.clear_error();
-    return createStringError(Error, "failed to write to \"%s\"",
-                             GeneratedUniqPath.c_str());
+    return llvm::make_error<AtomicFileWriteError>(
+        atomic_write_error::output_stream_error);
   }
 
   if (const std::error_code Error =
           sys::fs::rename(/*from=*/GeneratedUniqPath.c_str(),
                           /*to=*/FinalPath.str().c_str())) {
-    return createStringError(Error, "failed to rename file \"%s\" to \"%s\"",
-                             GeneratedUniqPath.c_str(),
-                             FinalPath.str().c_str());
+    return llvm::make_error<AtomicFileWriteError>(
+        atomic_write_error::failed_to_rename_temp_file);
   }
 
+  RemoveTmpFileOnFail.releaseFile();
   return Error::success();
 }
+
+char llvm::AtomicFileWriteError::ID;