Index: llvm/include/llvm/LTO/LTO.h =================================================================== --- llvm/include/llvm/LTO/LTO.h +++ llvm/include/llvm/LTO/LTO.h @@ -210,10 +210,14 @@ /// appends ".thinlto.bc" and writes the index to that path. If /// ShouldEmitImportsFiles is true it also writes a list of imported files to a /// similar path with ".imports" appended instead. +/// OnWrite is callback which receives module identifier and notifies LTO user +/// that index file for the module (and optionally imports file) was created. +using IndexWriteCallback = std::function; ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles, - std::string LinkedObjectsFile); + std::string LinkedObjectsFile, + IndexWriteCallback OnWrite); /// This class implements a resolution-based interface to LLVM's LTO /// functionality. It supports regular LTO, parallel LTO code generation and Index: llvm/lib/LTO/LTO.cpp =================================================================== --- llvm/lib/LTO/LTO.cpp +++ llvm/lib/LTO/LTO.cpp @@ -1033,16 +1033,18 @@ std::string LinkedObjectsFileName; std::unique_ptr LinkedObjectsFile; + lto::IndexWriteCallback OnWrite; + public: WriteIndexesThinBackend( Config &Conf, ModuleSummaryIndex &CombinedIndex, const StringMap &ModuleToDefinedGVSummaries, std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles, - std::string LinkedObjectsFileName) + std::string LinkedObjectsFileName, lto::IndexWriteCallback OnWrite) : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries), OldPrefix(OldPrefix), NewPrefix(NewPrefix), ShouldEmitImportsFiles(ShouldEmitImportsFiles), - LinkedObjectsFileName(LinkedObjectsFileName) {} + LinkedObjectsFileName(LinkedObjectsFileName), OnWrite(OnWrite) {} Error start( unsigned Task, BitcodeModule BM, @@ -1075,9 +1077,14 @@ return errorCodeToError(EC); WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); - if (ShouldEmitImportsFiles) - return errorCodeToError( - EmitImportsFiles(ModulePath, NewModulePath + ".imports", ImportList)); + if (ShouldEmitImportsFiles) { + EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports", ImportList); + if (EC) + return errorCodeToError(EC); + } + + if (OnWrite) + OnWrite(ModulePath); return Error::success(); } @@ -1088,13 +1095,14 @@ ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles, - std::string LinkedObjectsFile) { + std::string LinkedObjectsFile, + IndexWriteCallback OnWrite) { return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex, const StringMap &ModuleToDefinedGVSummaries, AddStreamFn AddStream, NativeObjectCache Cache) { return llvm::make_unique( Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix, - ShouldEmitImportsFiles, LinkedObjectsFile); + ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite); }; } Index: llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll =================================================================== --- llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll +++ llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll @@ -1,37 +1,40 @@ +; RUN: rm -f %t*.o.thinlto.bc +; RUN: rm -f %t*.o.imports + ; First generate bitcode with a module summary index for each file -; RUN: opt -module-summary %s -o %t.o +; RUN: opt -module-summary %s -o %t1.o ; RUN: opt -module-summary %p/Inputs/thinlto_emit_linked_objects.ll -o %t2.o +; RUN: opt %s -o %t3.o ; Next do the ThinLink step, specifying thinlto-index-only so that the gold ; plugin exits after generating individual indexes. The objects the linker ; decided to include in the link should be emitted into the file specified -; after 'thinlto-index-only='. In this version of the test, only %t.o will +; after 'thinlto-index-only='. In this version of the test, only %t1.o will ; be included in the link, and not %t2.o since it is within ; a library (--start-lib/--end-lib pair) and not strongly referenced. ; Note that the support for detecting this is in gold v1.12. -; RUN: rm -f %t.o.thinlto.bc -; RUN: rm -f %t2.o.thinlto.bc -; RUN: rm -f %t.o.imports -; RUN: rm -f %t2.o.imports ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \ ; RUN: --plugin-opt=thinlto \ -; RUN: --plugin-opt=thinlto-index-only=%t3 \ +; RUN: --plugin-opt=thinlto-index-only=%t.index \ ; RUN: --plugin-opt=thinlto-emit-imports-files \ ; RUN: -m elf_x86_64 \ ; RUN: -o %t4 \ -; RUN: %t.o \ +; RUN: %t1.o %t3.o \ ; RUN: --start-lib %t2.o --end-lib ; Ensure that the expected output files are created, even for the file ; the linker decided not to include in the link. -; RUN: ls %t.o.thinlto.bc +; RUN: ls %t1.o.thinlto.bc ; RUN: ls %t2.o.thinlto.bc -; RUN: ls %t.o.imports +; RUN: ls %t3.o.thinlto.bc +; RUN: ls %t1.o.imports ; RUN: ls %t2.o.imports +; RUN: ls %t3.o.imports -; RUN: cat %t3 | FileCheck %s -; CHECK: thinlto_emit_linked_objects.ll.tmp.o +; RUN: cat %t.index | FileCheck %s +; CHECK: thinlto_emit_linked_objects.ll.tmp1.o ; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o +; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp3.o target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/tools/gold/gold-plugin.cpp =================================================================== --- llvm/tools/gold/gold-plugin.cpp +++ llvm/tools/gold/gold-plugin.cpp @@ -750,7 +750,7 @@ std::tie(OldPrefix, NewPrefix) = PrefixReplace.split(';'); } -static std::unique_ptr createLTO() { +static std::unique_ptr createLTO(IndexWriteCallback OnIndexWrite) { Config Conf; ThinBackend Backend; @@ -777,7 +777,7 @@ getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix); Backend = createWriteIndexesThinBackend( OldPrefix, NewPrefix, options::thinlto_emit_imports_files, - options::thinlto_linked_objects_file); + options::thinlto_linked_objects_file, OnIndexWrite); } Conf.OverrideTriple = options::triple; @@ -826,9 +826,9 @@ // final link. Frequently the distributed build system will want to // confirm that all expected outputs are created based on all of the // modules provided to the linker. -static void writeEmptyDistributedBuildOutputs(std::string &ModulePath, - std::string &OldPrefix, - std::string &NewPrefix) { +static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath, + const std::string &OldPrefix, + const std::string &NewPrefix) { std::string NewModulePath = getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix); std::error_code EC; @@ -865,7 +865,13 @@ // through Lto->run(). DenseMap> HandleToInputFile; - std::unique_ptr Lto = createLTO(); + // Owns string objects and tells if index file was already created. + StringMap ObjectToIndexFileState; + + std::unique_ptr Lto = + createLTO([&ObjectToIndexFileState](const std::string &Identifier) { + ObjectToIndexFileState[Identifier] = true; + }); std::string OldPrefix, NewPrefix; if (options::thinlto_index_only) @@ -873,29 +879,20 @@ std::string OldSuffix, NewSuffix; getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); - // Set for owning string objects used as buffer identifiers. - StringSet<> ObjectFilenames; for (claimed_file &F : Modules) { if (options::thinlto && !HandleToInputFile.count(F.leader_handle)) HandleToInputFile.insert(std::make_pair( F.leader_handle, llvm::make_unique(F.handle))); - const void *View = getSymbolsAndView(F); // In case we are thin linking with a minimized bitcode file, ensure // the module paths encoded in the index reflect where the backends // will locate the full bitcode files for compiling/importing. std::string Identifier = getThinLTOObjectFileName(F.name, OldSuffix, NewSuffix); - auto ObjFilename = ObjectFilenames.insert(Identifier); + auto ObjFilename = ObjectToIndexFileState.insert({Identifier, false}); assert(ObjFilename.second); - if (!View) { - if (options::thinlto_index_only) - // Write empty output files that may be expected by the distributed - // build system. - writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix); - continue; - } - addModule(*Lto, F, View, ObjFilename.first->first()); + if (const void *View = getSymbolsAndView(F)) + addModule(*Lto, F, View, ObjFilename.first->first()); } SmallString<128> Filename; @@ -932,6 +929,17 @@ check(Lto->run(AddStream, Cache)); + if (options::thinlto_index_only) { + // Write empty output files that may be expected by the distributed + // build system. + for (auto &Identifier : ObjectToIndexFileState) { + if (!Identifier.getValue()) { + writeEmptyDistributedBuildOutputs(Identifier.getKey(), OldPrefix, + NewPrefix); + } + } + } + if (options::TheOutputType == options::OT_DISABLE || options::TheOutputType == options::OT_BC_ONLY) return LDPS_OK; Index: llvm/tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- llvm/tools/llvm-lto2/llvm-lto2.cpp +++ llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -243,7 +243,7 @@ ThinBackend Backend; if (ThinLTODistributedIndexes) - Backend = createWriteIndexesThinBackend("", "", true, ""); + Backend = createWriteIndexesThinBackend("", "", true, "", {}); else Backend = createInProcessThinBackend(Threads); LTO Lto(std::move(Conf), std::move(Backend));