Index: include/llvm/LTO/Config.h =================================================================== --- include/llvm/LTO/Config.h +++ include/llvm/LTO/Config.h @@ -29,6 +29,7 @@ class raw_pwrite_stream; namespace lto { +class Output; /// LTO configuration. A linker can configure LTO by setting fields in this data /// structure and passing it to the lto::LTO constructor. @@ -186,13 +187,11 @@ bool UseInputModulePath = false); }; -/// This type defines a stream callback. A stream callback is used to add a -/// native object that is generated on the fly. The callee must set up and -/// return a output stream to write the native object to. +/// This type defines the callback to add a native object that is generated on +/// the fly. /// -/// Stream callbacks must be thread safe. -typedef std::function(unsigned Task)> - AddStreamFn; +/// Output callbacks must be thread safe. +typedef std::function(unsigned Task)> AddOutputFn; /// A derived class of LLVMContext that initializes itself according to a given /// Config object. The purpose of this class is to tie ownership of the Index: include/llvm/LTO/LTO.h =================================================================== --- include/llvm/LTO/LTO.h +++ include/llvm/LTO/LTO.h @@ -230,13 +230,22 @@ } }; +/// Abstract class representing a single Task output to be implemented by the +/// client of the LTO API. +class Output { +public: + // Return an allocated stream for the output, or null in case of failure. + virtual std::unique_ptr getStream() = 0; + virtual ~Output() = default; +}; + /// A ThinBackend defines what happens after the thin-link phase during ThinLTO. /// The details of this type definition aren't important; clients can only /// create a ThinBackend using one of the create*ThinBackend() functions below. typedef std::function( Config &C, ModuleSummaryIndex &CombinedIndex, StringMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream)> + AddOutputFn AddOutput)> ThinBackend; /// This ThinBackend runs the individual backend jobs in-process. @@ -269,7 +278,7 @@ /// and pass it and an array of symbol resolutions to the add() function. /// - Call the getMaxTasks() function to get an upper bound on the number of /// native object files that LTO may add to the link. -/// - Call the run() function. This function will use the supplied AddStream +/// - Call the run() function. This function will use the supplied AddOutput /// function to add up to getMaxTasks() native object files to the link. class LTO { friend InputFile; @@ -293,9 +302,9 @@ /// full description of tasks see LTOBackend.h. unsigned getMaxTasks() const; - /// Runs the LTO pipeline. This function calls the supplied AddStream function + /// Runs the LTO pipeline. This function calls the supplied AddOutput function /// to add native object files to the link. - Error run(AddStreamFn AddStream); + Error run(AddOutputFn AddOutput); private: Config Conf; @@ -371,8 +380,8 @@ Error addThinLTO(std::unique_ptr Input, ArrayRef Res); - Error runRegularLTO(AddStreamFn AddStream); - Error runThinLTO(AddStreamFn AddStream); + Error runRegularLTO(AddOutputFn AddOutput); + Error runThinLTO(AddOutputFn AddOutput); mutable bool CalledGetMaxTasks = false; }; Index: include/llvm/LTO/LTOBackend.h =================================================================== --- include/llvm/LTO/LTOBackend.h +++ include/llvm/LTO/LTOBackend.h @@ -34,12 +34,12 @@ namespace lto { /// Runs a regular LTO backend. -Error backend(Config &C, AddStreamFn AddStream, +Error backend(Config &C, AddOutputFn AddStream, unsigned ParallelCodeGenParallelismLevel, std::unique_ptr M); /// Runs a ThinLTO backend. -Error thinBackend(Config &C, unsigned Task, AddStreamFn AddStream, Module &M, +Error thinBackend(Config &C, unsigned Task, AddOutputFn AddStream, Module &M, ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -350,19 +350,19 @@ return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size(); } -Error LTO::run(AddStreamFn AddStream) { +Error LTO::run(AddOutputFn AddOutput) { // Invoke regular LTO if there was a regular LTO module to start with, // or if there are any hooks that the linker may have used to add // its own resolved symbols to the combined module. if (RegularLTO.HasModule || Conf.PreOptModuleHook || Conf.PostInternalizeModuleHook || Conf.PostOptModuleHook || Conf.PreCodeGenModuleHook) - if (auto E = runRegularLTO(AddStream)) + if (auto E = runRegularLTO(AddOutput)) return E; - return runThinLTO(AddStream); + return runThinLTO(AddOutput); } -Error LTO::runRegularLTO(AddStreamFn AddStream) { +Error LTO::runRegularLTO(AddOutputFn AddOutput) { if (Conf.PreOptModuleHook && !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule)) return Error(); @@ -388,7 +388,7 @@ !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule)) return Error(); - return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel, + return backend(Conf, AddOutput, RegularLTO.ParallelCodeGenParallelismLevel, std::move(RegularLTO.CombinedModule)); } @@ -397,14 +397,14 @@ protected: Config &Conf; ModuleSummaryIndex &CombinedIndex; - AddStreamFn AddStream; + AddOutputFn AddOutput; StringMap &ModuleToDefinedGVSummaries; public: ThinBackendProc(Config &Conf, ModuleSummaryIndex &CombinedIndex, - AddStreamFn AddStream, + AddOutputFn AddOutput, StringMap &ModuleToDefinedGVSummaries) - : Conf(Conf), CombinedIndex(CombinedIndex), AddStream(AddStream), + : Conf(Conf), CombinedIndex(CombinedIndex), AddOutput(AddOutput), ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries) {} virtual ~ThinBackendProc() {} @@ -424,13 +424,13 @@ InProcessThinBackend(Config &Conf, ModuleSummaryIndex &CombinedIndex, unsigned ThinLTOParallelismLevel, StringMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream) - : ThinBackendProc(Conf, CombinedIndex, AddStream, + AddOutputFn AddOutput) + : ThinBackendProc(Conf, CombinedIndex, AddOutput, ModuleToDefinedGVSummaries), BackendThreadPool(ThinLTOParallelismLevel) {} Error - runThinLTOBackendThread(AddStreamFn AddStream, unsigned Task, + runThinLTOBackendThread(AddOutputFn AddOutput, unsigned Task, MemoryBufferRef MBRef, ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, @@ -442,7 +442,7 @@ parseBitcodeFile(MBRef, BackendContext); assert(MOrErr && "Unable to load module in thread?"); - return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex, + return thinBackend(Conf, Task, AddOutput, **MOrErr, CombinedIndex, ImportList, DefinedGlobals, ModuleMap); } @@ -456,7 +456,7 @@ GVSummaryMapTy &DefinedGlobals, MapVector &ModuleMap) { Error E = - runThinLTOBackendThread(AddStream, Task, MBRef, CombinedIndex, + runThinLTOBackendThread(AddOutput, Task, MBRef, CombinedIndex, ImportList, DefinedGlobals, ModuleMap); if (E) { std::unique_lock L(ErrMu); @@ -483,10 +483,10 @@ ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) { return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex, StringMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream) { + AddOutputFn AddOutput) { return llvm::make_unique( Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries, - AddStream); + AddOutput); }; } @@ -500,10 +500,10 @@ public: WriteIndexesThinBackend(Config &Conf, ModuleSummaryIndex &CombinedIndex, StringMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, std::string OldPrefix, + AddOutputFn AddOutput, std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles, std::string LinkedObjectsFileName) - : ThinBackendProc(Conf, CombinedIndex, AddStream, + : ThinBackendProc(Conf, CombinedIndex, AddOutput, ModuleToDefinedGVSummaries), OldPrefix(OldPrefix), NewPrefix(NewPrefix), ShouldEmitImportsFiles(ShouldEmitImportsFiles), @@ -572,14 +572,14 @@ std::string LinkedObjectsFile) { return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex, StringMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream) { + AddOutputFn AddOutput) { return llvm::make_unique( - Conf, CombinedIndex, ModuleToDefinedGVSummaries, AddStream, OldPrefix, + Conf, CombinedIndex, ModuleToDefinedGVSummaries, AddOutput, OldPrefix, NewPrefix, ShouldEmitImportsFiles, LinkedObjectsFile); }; } -Error LTO::runThinLTO(AddStreamFn AddStream) { +Error LTO::runThinLTO(AddOutputFn AddOutput) { if (ThinLTO.ModuleMap.empty()) return Error(); @@ -622,7 +622,7 @@ [](StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes) {}); std::unique_ptr BackendProc = ThinLTO.Backend( - Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, AddStream); + Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, AddOutput); // Partition numbers for ThinLTO jobs start at 1 (see comments for // GlobalResolution in LTO.h). Task numbers, however, start at Index: lib/LTO/LTOBackend.cpp =================================================================== --- lib/LTO/LTOBackend.cpp +++ lib/LTO/LTOBackend.cpp @@ -19,6 +19,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/LegacyPassManager.h" +#include "llvm/LTO/LTO.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -144,12 +145,13 @@ return true; } -void codegen(Config &C, TargetMachine *TM, AddStreamFn AddStream, unsigned Task, +void codegen(Config &C, TargetMachine *TM, AddOutputFn AddOutput, unsigned Task, Module &M) { if (C.PreCodeGenModuleHook && !C.PreCodeGenModuleHook(Task, M)) return; - std::unique_ptr OS = AddStream(Task); + auto Output = AddOutput(Task); + std::unique_ptr OS = Output->getStream(); legacy::PassManager CodeGenPasses; if (TM->addPassesToEmitFile(CodeGenPasses, *OS, TargetMachine::CGFT_ObjectFile)) @@ -157,7 +159,7 @@ CodeGenPasses.run(M); } -void splitCodeGen(Config &C, TargetMachine *TM, AddStreamFn AddStream, +void splitCodeGen(Config &C, TargetMachine *TM, AddOutputFn AddOutput, unsigned ParallelCodeGenParallelismLevel, std::unique_ptr M) { ThreadPool CodegenThreadPool(ParallelCodeGenParallelismLevel); @@ -190,7 +192,7 @@ std::unique_ptr TM = createTargetMachine(C, MPartInCtx->getTargetTriple(), T); - codegen(C, TM.get(), AddStream, ThreadId, *MPartInCtx); + codegen(C, TM.get(), AddOutput, ThreadId, *MPartInCtx); }, // Pass BC using std::move to ensure that it get moved rather than // copied into the thread's context. @@ -214,7 +216,7 @@ } -Error lto::backend(Config &C, AddStreamFn AddStream, +Error lto::backend(Config &C, AddOutputFn AddOutput, unsigned ParallelCodeGenParallelismLevel, std::unique_ptr M) { Expected TOrErr = initAndLookupTarget(C, *M); @@ -228,14 +230,14 @@ return Error(); if (ParallelCodeGenParallelismLevel == 1) - codegen(C, TM.get(), AddStream, 0, *M); + codegen(C, TM.get(), AddOutput, 0, *M); else - splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel, + splitCodeGen(C, TM.get(), AddOutput, ParallelCodeGenParallelismLevel, std::move(M)); return Error(); } -Error lto::thinBackend(Config &Conf, unsigned Task, AddStreamFn AddStream, +Error lto::thinBackend(Config &Conf, unsigned Task, AddOutputFn AddOutput, Module &Mod, ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, @@ -281,6 +283,6 @@ if (!opt(Conf, TM.get(), Task, Mod, /*IsThinLto=*/true)) return Error(); - codegen(Conf, TM.get(), AddStream, Task, Mod); + codegen(Conf, TM.get(), AddOutput, Task, Mod); return Error(); } Index: tools/gold/gold-plugin.cpp =================================================================== --- tools/gold/gold-plugin.cpp +++ tools/gold/gold-plugin.cpp @@ -647,13 +647,11 @@ Cleanup.push_back(Filename.c_str()); } -/// Open a file and return the new file descriptor given a base input -/// file name, a flag indicating whether a temp file should be generated, -/// and an optional task id. The new filename generated is -/// returned in \p NewFilename. -static int openOutputFile(SmallString<128> InFilename, bool TempOutFile, - SmallString<128> &NewFilename, int TaskID = -1) { - int FD; +/// Return the desired output filename given a base input name, a flag +/// indicating whether a temp file should be generated, and an optional task id. +/// The new filename generated is returned in \p NewFilename. +static void getOutputFileName(SmallString<128> InFilename, bool TempOutFile, + SmallString<128> &NewFilename, int TaskID = -1) { if (TempOutFile) { std::error_code EC = sys::fs::createTemporaryFile("lto-llvm", "o", FD, NewFilename); @@ -664,12 +662,7 @@ NewFilename = InFilename; if (TaskID >= 0) NewFilename += utostr(TaskID); - std::error_code EC = - sys::fs::openFileForWrite(NewFilename, FD, sys::fs::F_None); - if (EC) - message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str()); } - return FD; } /// Add all required common symbols to M, which is expected to be the first @@ -723,6 +716,24 @@ NewPrefix = Split.second.str(); } +namespace { +// Define the LTOOutput handling +class LTOOutput : public lto::Output { + StringRef Path; + +public: + LTOOutput(StringRef Path) : Path(Path) {} + // Open the filename \p Path and allocate a stream. + std::unique_ptr getStream() override { + int FD; + std::error_code EC = sys::fs::openFileForWrite(Path, FD, sys::fs::F_None); + if (EC) + message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str()); + return llvm::make_unique(FD, true) + } +}; +} + static std::unique_ptr createLTO() { Config Conf; ThinBackend Backend; @@ -814,7 +825,7 @@ } SmallString<128> Filename; - // Note that openOutputFile will append a unique ID for each task + // Note that getOutputFileName will append a unique ID for each task if (!options::obj_path.empty()) Filename = options::obj_path; else if (options::TheOutputType == options::OT_SAVE_TEMPS) @@ -825,15 +836,15 @@ std::vector IsTemporary(MaxTasks); std::vector> Filenames(MaxTasks); - auto AddStream = [&](size_t Task) { - int FD = openOutputFile(Filename, /*TempOutFile=*/!SaveTemps, - Filenames[Task], MaxTasks > 1 ? Task : -1); + auto AddOutput = [&](size_t Task) { + auto &OutputName = Filenames[Task]; + getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, OutputName, + MaxTasks > 1 ? Task : -1); IsTemporary[Task] = !SaveTemps; - - return llvm::make_unique(FD, true); + return llvm::make_unique(OutputName); }; - check(Lto->run(AddStream)); + check(Lto->run(AddOutput)); if (options::TheOutputType == options::OT_DISABLE || options::TheOutputType == options::OT_BC_ONLY) Index: tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- tools/llvm-lto2/llvm-lto2.cpp +++ tools/llvm-lto2/llvm-lto2.cpp @@ -74,6 +74,22 @@ return T(); } +namespace { +// Define the LTOOutput handling +class LTOOutput : public lto::Output { + StringRef Path; + +public: + LTOOutput(StringRef Path) : Path(Path) {} + std::unique_ptr getStream() override { + std::error_code EC; + auto S = llvm::make_unique(Path, EC, sys::fs::F_None); + check(EC, Path); + return std::move(S); + } +}; +} + int main(int argc, char **argv) { InitializeAllTargets(); InitializeAllTargetMCs(); @@ -156,13 +172,10 @@ if (HasErrors) return 1; - auto AddStream = [&](size_t Task) { + auto AddOutput = [&](size_t Task) { std::string Path = OutputFilename + "." + utostr(Task); - std::error_code EC; - auto S = llvm::make_unique(Path, EC, sys::fs::F_None); - check(EC, Path); - return S; + return llvm::make_unique(Path); }; - check(Lto.run(AddStream), "LTO::run failed"); + check(Lto.run(AddOutput), "LTO::run failed"); }