Index: include/llvm/LTO/Config.h =================================================================== --- include/llvm/LTO/Config.h +++ include/llvm/LTO/Config.h @@ -52,6 +52,10 @@ unsigned OptLevel = 2; bool DisableVerify = false; + /// True if all input files should be handed to a ThinLTO backend, and + /// not inferred from presence of module summary. + bool ForceThinLTO = false; + /// Setting this field will replace target triples in input files with this /// triple. std::string OverrideTriple; @@ -138,6 +142,7 @@ RelocModel(std::move(X.RelocModel)), CodeModel(std::move(X.CodeModel)), CGOptLevel(std::move(X.CGOptLevel)), OptLevel(std::move(X.OptLevel)), DisableVerify(std::move(X.DisableVerify)), + ForceThinLTO(std::move(X.ForceThinLTO)), OverrideTriple(std::move(X.OverrideTriple)), DefaultTriple(std::move(X.DefaultTriple)), ShouldDiscardValueNames(std::move(X.ShouldDiscardValueNames)), @@ -161,6 +166,7 @@ CGOptLevel = std::move(X.CGOptLevel); OptLevel = std::move(X.OptLevel); DisableVerify = std::move(X.DisableVerify); + ForceThinLTO = std::move(X.ForceThinLTO); OverrideTriple = std::move(X.OverrideTriple); DefaultTriple = std::move(X.DefaultTriple); ShouldDiscardValueNames = std::move(X.ShouldDiscardValueNames); Index: include/llvm/LTO/LTO.h =================================================================== --- include/llvm/LTO/LTO.h +++ include/llvm/LTO/LTO.h @@ -292,6 +292,13 @@ /// InputFile::symbols(). Error add(std::unique_ptr Obj, ArrayRef Res); + /// Add an input file that the linker decided not to include in the link. + /// This could happen for objects from archive libraries that weren't + /// strongly referenced. Currently this is used to ensure they get + /// (empty) individual module index files, so that a distributed build + /// system can validate the list of expected outputs. + Error addNotLinkedModule(StringRef ModulePath); + /// Returns an upper bound on the number of tasks that the client may expect. /// This may only be called after all IR object files have been added. For a /// full description of tasks see LTOBackend.h. @@ -363,6 +370,9 @@ // Global mapping from mangled symbol names to resolutions. StringMap GlobalResolutions; + // List of modules that the linker decided not to include in the link. + std::vector NotLinkedModules; + void writeToResolutionFile(InputFile *Input, ArrayRef Res); void addSymbolToGlobalRes(object::IRObjectFile *Obj, Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -240,9 +240,8 @@ M.setTargetTriple(Conf.DefaultTriple); MemoryBufferRef MBRef = Input->Obj->getMemoryBufferRef(); - bool HasThinLTOSummary = hasGlobalValueSummary(MBRef, Conf.DiagHandler); - if (HasThinLTOSummary) + if (Conf.ForceThinLTO || hasGlobalValueSummary(MBRef, Conf.DiagHandler)) return addThinLTO(std::move(Input), Res); else return addRegularLTO(std::move(Input), Res); @@ -345,6 +344,11 @@ return Error(); } +Error LTO::addNotLinkedModule(StringRef ModulePath) { + NotLinkedModules.push_back(ModulePath); + return Error(); +} + unsigned LTO::getMaxTasks() const { CalledGetMaxTasks = true; return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size(); @@ -411,6 +415,7 @@ virtual Error start(unsigned Task, MemoryBufferRef MBRef, const FunctionImporter::ImportMapTy &ImportList, MapVector &ModuleMap) = 0; + virtual Error startNotLinkedFileHandling(StringRef ModulePath) = 0; virtual Error wait() = 0; }; @@ -471,6 +476,12 @@ return Error(); } + // For an in process backend we ignore any modules the linker decided + // not to include in the link. + Error startNotLinkedFileHandling(StringRef ModulePath) override { + return Error(); + } + Error wait() override { BackendThreadPool.wait(); if (Err) @@ -529,15 +540,14 @@ return NewPath.str(); } - Error start(unsigned Task, MemoryBufferRef MBRef, - const FunctionImporter::ImportMapTy &ImportList, - MapVector &ModuleMap) override { - StringRef ModulePath = MBRef.getBufferIdentifier(); + Error WriteIndexFiles(StringRef ModulePath, + const FunctionImporter::ImportMapTy &ImportList, + bool IsLinked) { std::string NewModulePath = getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix); std::error_code EC; - if (!LinkedObjectsFileName.empty()) { + if (IsLinked && !LinkedObjectsFileName.empty()) { if (!LinkedObjectsFile) { LinkedObjectsFile = llvm::make_unique( LinkedObjectsFileName, EC, sys::fs::OpenFlags::F_None); @@ -548,8 +558,9 @@ } std::map ModuleToSummariesForIndex; - gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, - ImportList, ModuleToSummariesForIndex); + if (IsLinked) + gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, + ImportList, ModuleToSummariesForIndex); raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, sys::fs::OpenFlags::F_None); @@ -563,6 +574,21 @@ return Error(); } + Error start(unsigned Task, MemoryBufferRef MBRef, + const FunctionImporter::ImportMapTy &ImportList, + MapVector &ModuleMap) override { + return WriteIndexFiles(MBRef.getBufferIdentifier(), ImportList, + /* IsLinked = */ true); + } + + // For a distributed build we ensure that the expected output files + // are created even for those files the linker decided not to include. + Error startNotLinkedFileHandling(StringRef ModulePath) override { + FunctionImporter::ImportMapTy EmptyImportList; + return WriteIndexFiles(ModulePath, EmptyImportList, + /* IsLinked = */ false); + } + Error wait() override { return Error(); } }; @@ -641,5 +667,10 @@ ++Partition; } + for (auto ModPath : NotLinkedModules) { + if (Error E = BackendProc->startNotLinkedFileHandling(ModPath)) + return E; + } + return BackendProc->wait(); } Index: test/ThinLTO/X86/Inputs/emit_imports.ll =================================================================== --- test/ThinLTO/X86/Inputs/emit_imports.ll +++ test/ThinLTO/X86/Inputs/emit_imports.ll @@ -1,3 +1,6 @@ +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + define void @g() { entry: ret void Index: test/ThinLTO/X86/Inputs/empty.ll =================================================================== --- /dev/null +++ test/ThinLTO/X86/Inputs/empty.ll @@ -0,0 +1,2 @@ +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" Index: test/ThinLTO/X86/emit_imports.ll =================================================================== --- test/ThinLTO/X86/emit_imports.ll +++ test/ThinLTO/X86/emit_imports.ll @@ -1,7 +1,11 @@ ; RUN: opt -module-summary %s -o %t1.bc ; RUN: opt -module-summary %p/Inputs/emit_imports.ll -o %t2.bc -; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc -; RUN: llvm-lto -thinlto-action=emitimports -thinlto-index %t.index.bc %t1.bc %t2.bc +; Include a file without a module summary index, to ensure that the expected +; output files are created regardless, for a distributed build system. +; RUN: opt %p/Inputs/empty.ll -o %t3.bc +; RUN: rm -f %t3.bc.imports +; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc %t3.bc +; RUN: llvm-lto -thinlto-action=emitimports -thinlto-index %t.index.bc %t1.bc %t2.bc %t3.bc ; The imports file for this module contains the bitcode file for ; Inputs/emit_imports.ll @@ -12,6 +16,21 @@ ; The imports file for Input/emit_imports.ll is empty as it does not import anything. ; RUN: cat %t2.bc.imports | count 0 +; The imports file for Input/empty.ll is empty but should exist. +; RUN: cat %t3.bc.imports | count 0 + +; RUN: rm -f %t3.bc.thinlto.bc %t3.bc.imports +; RUN: llvm-lto2 %t1.bc %t2.bc %t3.bc -o %t.o -save-temps \ +; RUN: -thinlto-index-only \ +; RUN: -r=%t1.bc,g, \ +; RUN: -r=%t1.bc,f,px \ +; RUN: -r=%t2.bc,g,px +; RUN: ls %t3.bc.thinlto.bc +; RUN: ls %t3.bc.imports + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + declare void @g(...) define void @f() { Index: test/tools/gold/X86/thinlto_emit_imports.ll =================================================================== --- test/tools/gold/X86/thinlto_emit_imports.ll +++ test/tools/gold/X86/thinlto_emit_imports.ll @@ -1,13 +1,17 @@ ; Generate summary sections and test gold handling. ; RUN: opt -module-summary %s -o %t.o ; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o +; Include a file without a module summary index, to ensure that the expected +; output files are created regardless, for a distributed build system. +; RUN: opt %p/Inputs/thinlto_empty.ll -o %t3.o ; Ensure gold generates imports files if requested for distributed backends. +; RUN: rm -f %t3.o.imports %t3.o.thinlto.bc ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=thinlto \ ; RUN: --plugin-opt=thinlto-index-only \ ; RUN: --plugin-opt=thinlto-emit-imports-files \ -; RUN: -shared %t.o %t2.o -o %t3 +; RUN: -shared %t.o %t2.o %t3.o -o %t4 ; The imports file for this module contains the bitcode file for ; Inputs/thinlto.ll @@ -18,6 +22,10 @@ ; The imports file for Input/thinlto.ll is empty as it does not import anything. ; RUN: cat %t2.o.imports | count 0 +; These files should be created even for the input without a summary index. +; RUN: ls %t3.o.imports +; RUN: ls %t3.o.thinlto.bc + declare void @g(...) define void @f() { Index: test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll =================================================================== --- test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll +++ test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll @@ -9,14 +9,26 @@ ; be included in the link, and not %t2.o since it is within ; a library (--start-lib/--end-lib pair) and not strongly referenced. ; Note that the support for detecting this is in gold v1.12. +; RUN: rm -f %t.o.thinlto.bc +; RUN: rm -f %t2.o.thinlto.bc +; RUN: rm -f %t.o.imports +; RUN: rm -f %t2.o.imports ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=thinlto \ ; RUN: --plugin-opt=thinlto-index-only=%t3 \ +; RUN: --plugin-opt=thinlto-emit-imports-files \ ; RUN: -m elf_x86_64 \ ; RUN: -o %t4 \ ; RUN: %t.o \ ; RUN: --start-lib %t2.o --end-lib +; Ensure that the expected output files are created, even for the file +; the linker decided not to include in the link. +; RUN: ls %t.o.thinlto.bc +; RUN: ls %t2.o.thinlto.bc +; RUN: ls %t.o.imports +; RUN: ls %t2.o.imports + ; RUN: cat %t3 | FileCheck %s ; CHECK: thinlto_emit_linked_objects.ll.tmp.o ; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o Index: tools/gold/gold-plugin.cpp =================================================================== --- tools/gold/gold-plugin.cpp +++ tools/gold/gold-plugin.cpp @@ -572,7 +572,8 @@ toString(ObjOrErr.takeError()).c_str()); InputFile &Obj = **ObjOrErr; - bool HasThinLTOSummary = + bool IsThinLTO = + options::thinlto_index_only || hasGlobalValueSummary(Obj.getMemoryBufferRef(), diagnosticHandler); unsigned SymNum = 0; @@ -619,8 +620,7 @@ (IsExecutable || !Res.DefaultVisibility)) R.FinalDefinitionInLinkageUnit = true; - if ((ObjSym.getFlags() & object::BasicSymbolRef::SF_Common) && - !HasThinLTOSummary) { + if ((ObjSym.getFlags() & object::BasicSymbolRef::SF_Common) && !IsThinLTO) { // We ignore gold's resolution for common symbols. A common symbol with // the correct size and alignment is added to the module by the pre-opt // module hook if any common symbol prevailed. @@ -766,6 +766,7 @@ Backend = createWriteIndexesThinBackend( OldPrefix, NewPrefix, options::thinlto_emit_imports_files, options::thinlto_linked_objects_file); + Conf.ForceThinLTO = true; } Conf.OverrideTriple = options::triple; @@ -823,8 +824,11 @@ for (claimed_file &F : Modules) { PluginInputFile InputFile(F.handle); const void *View = getSymbolsAndView(F); - if (!View) + if (!View) { + check(Lto->addNotLinkedModule(F.name), + std::string("Failed to link module ") + F.name); continue; + } addModule(*Lto, F, View); } Index: tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- tools/llvm-lto2/llvm-lto2.cpp +++ tools/llvm-lto2/llvm-lto2.cpp @@ -33,6 +33,10 @@ static cl::opt SaveTemps("save-temps", cl::desc("Save temporary files")); +static cl::opt + ThinLTOIndexOnly("thinlto-index-only", + cl::desc("Emit individual index files and exit")); + static cl::list SymbolResolutions( "r", cl::desc("Specify a symbol resolution: filename,symbolname,resolution\n" @@ -135,7 +139,16 @@ check(Conf.addSaveTemps(OutputFilename + "."), "Config::addSaveTemps failed"); - LTO Lto(std::move(Conf)); + ThinBackend Backend; + if (ThinLTOIndexOnly) { + // FIXME: Support passing in old/new prefix and linked object file. + Backend = createWriteIndexesThinBackend( + /* OldPrefix = */ "", /* NewPrefix = */ "", + /* ShouldEmitImportsFiles = */ true, /* LinkedObjectsFile = */ ""); + Conf.ForceThinLTO = true; + } + + LTO Lto(std::move(Conf), Backend); bool HasErrors = false; for (std::string F : InputFilenames) {