diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt --- a/clang/docs/tools/clang-formatted-files.txt +++ b/clang/docs/tools/clang-formatted-files.txt @@ -4745,7 +4745,6 @@ llvm/include/llvm/IR/StructuralHash.h llvm/include/llvm/IR/TrackingMDRef.h llvm/include/llvm/IR/UseListOrder.h -llvm/include/llvm/LTO/Caching.h llvm/include/llvm/LTO/SummaryBasedOptimizations.h llvm/include/llvm/MC/MCAsmInfoCOFF.h llvm/include/llvm/MC/MCAsmInfoDarwin.h @@ -4852,6 +4851,7 @@ llvm/include/llvm/Support/BinaryStreamWriter.h llvm/include/llvm/Support/BuryPointer.h llvm/include/llvm/Support/CachePruning.h +llvm/include/llvm/Support/Caching.h llvm/include/llvm/Support/CFGDiff.h llvm/include/llvm/Support/CFGUpdate.h llvm/include/llvm/Support/CodeGenCoverage.h diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1561,7 +1561,7 @@ return; auto AddStream = [&](size_t Task) { - return std::make_unique(std::move(OS)); + return std::make_unique(std::move(OS)); }; lto::Config Conf; if (CGOpts.SaveTempsFilePrefix != "") { diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -20,10 +20,10 @@ #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -164,16 +164,17 @@ // The /lldltocache option specifies the path to a directory in which to cache // native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->ltoCache.empty()) - cache = check(lto::localCache( - config->ltoCache, [&](size_t task, std::unique_ptr mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->ltoCache, + [&](size_t task, std::unique_ptr mb) { + files[task] = std::move(mb); + })); checkError(ltoObj->run( [&](size_t task) { - return std::make_unique( + return std::make_unique( std::make_unique(buf[task])); }, cache)); diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -23,10 +23,10 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -304,18 +304,18 @@ // The --thinlto-cache-dir option specifies the path to a directory in which // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->thinLTOCacheDir.empty()) - cache = check( - lto::localCache(config->thinLTOCacheDir, - [&](size_t task, std::unique_ptr mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr mb) { + files[task] = std::move(mb); + })); if (!bitcodeFiles.empty()) checkError(ltoObj->run( [&](size_t task) { - return std::make_unique( + return std::make_unique( std::make_unique(buf[task])); }, cache)); diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -17,9 +17,9 @@ #include "lld/Common/ErrorHandler.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" @@ -105,17 +105,17 @@ // The -cache_path_lto option specifies the path to a directory in which // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->thinLTOCacheDir.empty()) - cache = check( - lto::localCache(config->thinLTOCacheDir, - [&](size_t task, std::unique_ptr mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr mb) { + files[task] = std::move(mb); + })); checkError(ltoObj->run( [&](size_t task) { - return std::make_unique( + return std::make_unique( std::make_unique(buf[task])); }, cache)); diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp --- a/lld/wasm/LTO.cpp +++ b/lld/wasm/LTO.cpp @@ -19,10 +19,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -127,17 +127,17 @@ // The --thinlto-cache-dir option specifies the path to a directory in which // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->thinLTOCacheDir.empty()) - cache = check( - lto::localCache(config->thinLTOCacheDir, - [&](size_t task, std::unique_ptr mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr mb) { + files[task] = std::move(mb); + })); checkError(ltoObj->run( [&](size_t task) { - return std::make_unique( + return std::make_unique( std::make_unique(buf[task])); }, cache)); diff --git a/llvm/include/llvm/LTO/Caching.h b/llvm/include/llvm/LTO/Caching.h deleted file mode 100644 --- a/llvm/include/llvm/LTO/Caching.h +++ /dev/null @@ -1,38 +0,0 @@ -//===- Caching.h - LLVM Link Time Optimizer Configuration -----------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines the localCache function, which allows clients to add a -// filesystem cache to ThinLTO. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LTO_CACHING_H -#define LLVM_LTO_CACHING_H - -#include "llvm/LTO/LTO.h" - -namespace llvm { -namespace lto { - -/// This type defines the callback to add a pre-existing native object file -/// (e.g. in a cache). -/// -/// Buffer callbacks must be thread safe. -using AddBufferFn = - std::function MB)>; - -/// Create a local file system cache which uses the given cache directory and -/// file callback. This function also creates the cache directory if it does not -/// already exist. -Expected localCache(StringRef CacheDirectoryPath, - AddBufferFn AddBuffer); - -} // namespace lto -} // namespace llvm - -#endif diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -21,6 +21,7 @@ #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/LTO/Config.h" #include "llvm/Object/IRSymtab.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/Error.h" #include "llvm/Support/thread.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" @@ -187,40 +188,6 @@ } }; -/// This class wraps an output stream for a native object. Most clients should -/// just be able to return an instance of this base class from the stream -/// callback, but if a client needs to perform some action after the stream is -/// written to, that can be done by deriving from this class and overriding the -/// destructor. -class NativeObjectStream { -public: - NativeObjectStream(std::unique_ptr OS) : OS(std::move(OS)) {} - std::unique_ptr OS; - virtual ~NativeObjectStream() = default; -}; - -/// This type defines the callback to add a native object that is generated on -/// the fly. -/// -/// Stream callbacks must be thread safe. -using AddStreamFn = - std::function(unsigned Task)>; - -/// This is the type of a native object cache. To request an item from the -/// cache, pass a unique string as the Key. For hits, the cached file will be -/// added to the link and this function will return AddStreamFn(). For misses, -/// the cache will return a stream callback which must be called at most once to -/// produce content for the stream. The native object stream produced by the -/// stream callback will add the file to the link after the stream is written -/// to. -/// -/// Clients generally look like this: -/// -/// if (AddStreamFn AddStream = Cache(Task, Key)) -/// ProduceContent(AddStream); -using NativeObjectCache = - std::function; - /// A ThinBackend defines what happens after the thin-link phase during ThinLTO. /// The details of this type definition aren't important; clients can only /// create a ThinBackend using one of the create*ThinBackend() functions below. diff --git a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h --- a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -176,7 +176,7 @@ /// created using the \p AddStream callback. Returns true on success. /// /// Calls \a verifyMergedModuleOnce(). - bool compileOptimized(lto::AddStreamFn AddStream, unsigned ParallelismLevel); + bool compileOptimized(AddStreamFn AddStream, unsigned ParallelismLevel); /// Enable the Freestanding mode: indicate that the optimizer should not /// assume builtins are present on the target. diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/Caching.h @@ -0,0 +1,76 @@ +//===- Caching.h - LLVM File Cache Handling Configuration -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the localCache function, which allows clients to add a +// filesystem cache. This is used by ThinLTO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CACHING_H +#define LLVM_SUPPORT_CACHING_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/thread.h" + +namespace llvm { + +/// This class wraps an output stream for a native object. Most clients should +/// just be able to return an instance of this base class from the stream +/// callback, but if a client needs to perform some action after the stream is +/// written to, that can be done by deriving from this class and overriding the +/// destructor. +class NativeObjectStream { +public: + NativeObjectStream(std::unique_ptr OS) + : OS(std::move(OS)) {} + std::unique_ptr OS; + virtual ~NativeObjectStream() = default; +}; + +/// This type defines the callback to add a native object that is generated on +/// the fly. +/// +/// Stream callbacks must be thread safe. +using AddStreamFn = + std::function(unsigned Task)>; + +/// This is the type of a native object cache. To request an item from the +/// cache, pass a unique string as the Key. For hits, the cached file will be +/// added to the link and this function will return AddStreamFn(). For misses, +/// the cache will return a stream callback which must be called at most once to +/// produce content for the stream. The native object stream produced by the +/// stream callback will add the file to the link after the stream is written +/// to. +/// +/// Clients generally look like this: +/// +/// if (AddStreamFn AddStream = Cache(Task, Key)) +/// ProduceContent(AddStream); +using NativeObjectCache = + std::function; + +/// This type defines the callback to add a pre-existing native object file +/// (e.g. in a cache). +/// +/// Buffer callbacks must be thread safe. +using AddBufferFn = + std::function MB)>; + +/// Create a local file system cache which uses the given cache name, temporary +/// file prefix, cache directory and file callback. This function also creates +/// the cache directory if it does not already exist. The cache name appears in +/// error messages for errors during caching. The temporary file prefix is used +/// in the temporary file naming scheme used when writing files atomically. +Expected localCache(Twine CacheNameRef, + Twine TempFilePrefixRef, + Twine CacheDirectoryPathRef, + AddBufferFn AddBuffer); +} // namespace llvm + +#endif diff --git a/llvm/lib/LTO/CMakeLists.txt b/llvm/lib/LTO/CMakeLists.txt --- a/llvm/lib/LTO/CMakeLists.txt +++ b/llvm/lib/LTO/CMakeLists.txt @@ -1,5 +1,4 @@ add_llvm_component_library(LLVMLTO - Caching.cpp LTO.cpp LTOBackend.cpp LTOModule.cpp diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp --- a/llvm/lib/LTO/LTOCodeGenerator.cpp +++ b/llvm/lib/LTO/LTOCodeGenerator.cpp @@ -245,8 +245,7 @@ // make unique temp output file to put generated code SmallString<128> Filename; - auto AddStream = - [&](size_t Task) -> std::unique_ptr { + auto AddStream = [&](size_t Task) -> std::unique_ptr { StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o"); int FD; @@ -255,7 +254,7 @@ if (EC) emitError(EC.message()); - return std::make_unique( + return std::make_unique( std::make_unique(FD, true)); }; @@ -557,7 +556,7 @@ return true; } -bool LTOCodeGenerator::compileOptimized(lto::AddStreamFn AddStream, +bool LTOCodeGenerator::compileOptimized(AddStreamFn AddStream, unsigned ParallelismLevel) { if (!this->determineTarget()) return false; diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -123,6 +123,7 @@ BranchProbability.cpp BuryPointer.cpp CachePruning.cpp + Caching.cpp circular_raw_ostream.cpp Chrono.cpp COM.cpp diff --git a/llvm/lib/LTO/Caching.cpp b/llvm/lib/Support/Caching.cpp rename from llvm/lib/LTO/Caching.cpp rename to llvm/lib/Support/Caching.cpp --- a/llvm/lib/LTO/Caching.cpp +++ b/llvm/lib/Support/Caching.cpp @@ -1,4 +1,4 @@ -//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===// +//===-Caching.cpp - LLVM File Cache Handling ------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,11 +6,11 @@ // //===----------------------------------------------------------------------===// // -// This file implements the Caching for ThinLTO. +// This file implements the Caching used by ThinLTO. // //===----------------------------------------------------------------------===// -#include "llvm/LTO/Caching.h" +#include "llvm/Support/Caching.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" @@ -26,13 +26,20 @@ #endif using namespace llvm; -using namespace llvm::lto; -Expected lto::localCache(StringRef CacheDirectoryPath, - AddBufferFn AddBuffer) { - if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath)) +Expected llvm::localCache(Twine CacheNameRef, + Twine TempFilePrefixRef, + Twine CacheDirectoryPathRef, + AddBufferFn AddBuffer) { + if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPathRef)) return errorCodeToError(EC); + // Create local copies which are safely captured-by-copy in lambdas + SmallString<64> CacheName, TempFilePrefix, CacheDirectoryPath; + CacheNameRef.toVector(CacheName); + TempFilePrefixRef.toVector(TempFilePrefix); + CacheDirectoryPathRef.toVector(CacheDirectoryPath); + return [=](unsigned Task, StringRef Key) -> AddStreamFn { // This choice of file name allows the cache to be pruned (see pruneCache() // in include/llvm/Support/CachePruning.h). @@ -134,12 +141,13 @@ return [=](size_t Task) -> std::unique_ptr { // Write to a temporary to avoid race condition SmallString<64> TempFilenameModel; - sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o"); + sys::path::append(TempFilenameModel, CacheDirectoryPath, + TempFilePrefix + "-%%%%%%.tmp.o"); Expected Temp = sys::fs::TempFile::create( TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write); if (!Temp) { errs() << "Error: " << toString(Temp.takeError()) << "\n"; - report_fatal_error("ThinLTO: Can't get a temporary file"); + report_fatal_error(CacheName + ": Can't get a temporary file"); } // This CacheStream will move the temporary file into the cache when done. diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp --- a/llvm/tools/gold/gold-plugin.cpp +++ b/llvm/tools/gold/gold-plugin.cpp @@ -19,11 +19,11 @@ #include "llvm/Config/llvm-config.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/Error.h" #include "llvm/Remarks/HotnessThresholdParser.h" #include "llvm/Support/CachePruning.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Host.h" @@ -1081,12 +1081,11 @@ size_t MaxTasks = Lto->getMaxTasks(); std::vector, bool>> Files(MaxTasks); - auto AddStream = - [&](size_t Task) -> std::unique_ptr { + auto AddStream = [&](size_t Task) -> std::unique_ptr { Files[Task].second = !SaveTemps; int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps, Files[Task].first, Task); - return std::make_unique( + return std::make_unique( std::make_unique(FD, true)); }; @@ -1096,7 +1095,7 @@ NativeObjectCache Cache; if (!options::cache_dir.empty()) - Cache = check(localCache(options::cache_dir, AddBuffer)); + Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer)); check(Lto->run(AddStream, Cache)); diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp --- a/llvm/tools/llvm-lto/llvm-lto.cpp +++ b/llvm/tools/llvm-lto/llvm-lto.cpp @@ -1097,8 +1097,7 @@ error("writing merged module failed."); } - auto AddStream = - [&](size_t Task) -> std::unique_ptr { + auto AddStream = [&](size_t Task) -> std::unique_ptr { std::string PartFilename = OutputFilename; if (Parallelism != 1) PartFilename += "." + utostr(Task); @@ -1108,7 +1107,7 @@ std::make_unique(PartFilename, EC, sys::fs::OF_None); if (EC) error("error opening the file '" + PartFilename + "': " + EC.message()); - return std::make_unique(std::move(S)); + return std::make_unique(std::move(S)); }; if (!CodeGen.compileOptimized(AddStream, Parallelism)) diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -19,10 +19,10 @@ #include "llvm/CodeGen/CommandFlags.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/LTO.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Remarks/HotnessThresholdParser.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/InitLLVM.h" @@ -362,14 +362,13 @@ if (HasErrors) return 1; - auto AddStream = - [&](size_t Task) -> std::unique_ptr { + auto AddStream = [&](size_t Task) -> std::unique_ptr { std::string Path = OutputFilename + "." + utostr(Task); std::error_code EC; auto S = std::make_unique(Path, EC, sys::fs::OF_None); check(EC, Path); - return std::make_unique(std::move(S)); + return std::make_unique(std::move(S)); }; auto AddBuffer = [&](size_t Task, std::unique_ptr MB) { @@ -378,7 +377,8 @@ NativeObjectCache Cache; if (!CacheDir.empty()) - Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache"); + Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer), + "failed to create cache"); check(Lto.run(AddStream, Cache), "LTO::run failed"); return 0; diff --git a/llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn --- a/llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn @@ -22,7 +22,6 @@ "//llvm/lib/Transforms/Utils", ] sources = [ - "Caching.cpp", "LTO.cpp", "LTOBackend.cpp", "LTOCodeGenerator.cpp", diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn --- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn @@ -54,6 +54,7 @@ "COM.cpp", "CRC.cpp", "CachePruning.cpp", + "Caching.cpp", "Chrono.cpp", "CodeGenCoverage.cpp", "CommandLine.cpp",