Index: ELF/Driver.h =================================================================== --- ELF/Driver.h +++ ELF/Driver.h @@ -14,6 +14,7 @@ #include "lld/Core/LLVM.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/Option/ArgList.h" #include "llvm/Support/raw_ostream.h" Index: ELF/Driver.cpp =================================================================== --- ELF/Driver.cpp +++ ELF/Driver.cpp @@ -19,6 +19,7 @@ #include "Writer.h" #include "lld/Driver/Driver.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Path.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" Index: ELF/Error.h =================================================================== --- ELF/Error.h +++ ELF/Error.h @@ -12,6 +12,12 @@ #include "lld/Core/LLVM.h" +namespace llvm { + +class DiagnosticInfo; + +} + namespace lld { namespace elf { @@ -52,6 +58,8 @@ fatal(EO.getError().message(), Prefix); } +void diagHandler(const llvm::DiagnosticInfo &DI); + } // namespace elf } // namespace lld Index: ELF/Error.cpp =================================================================== --- ELF/Error.cpp +++ ELF/Error.cpp @@ -11,6 +11,8 @@ #include "Config.h" #include "llvm/ADT/Twine.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/Support/raw_ostream.h" namespace lld { @@ -50,5 +52,12 @@ fatal(EC.message()); } +void diagHandler(const llvm::DiagnosticInfo &DI) { + llvm::DiagnosticPrinterRawOStream DPOS(llvm::errs()); + DI.print(DPOS); + if (DI.getKind() == llvm::DS_Error) + exit(1); +} + } // namespace elf } // namespace lld Index: ELF/InputFiles.h =================================================================== --- ELF/InputFiles.h +++ ELF/InputFiles.h @@ -221,6 +221,7 @@ ArrayRef getSymbols() { return Symbols; } static bool shouldSkip(uint32_t Flags); std::unique_ptr Obj; + bool ThinLto; private: std::vector Symbols; Index: ELF/InputFiles.cpp =================================================================== --- ELF/InputFiles.cpp +++ ELF/InputFiles.cpp @@ -14,6 +14,7 @@ #include "SymbolTable.h" #include "Symbols.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -592,6 +593,7 @@ template void BitcodeFile::parse(DenseSet &ComdatGroups) { + ThinLto = hasGlobalValueSummary(MB, diagHandler); Obj = check(IRObjectFile::create(MB, Driver->Context)); const Module &M = Obj->getModule(); Index: ELF/LTO.h =================================================================== --- ELF/LTO.h +++ ELF/LTO.h @@ -25,7 +25,15 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/Linker/IRMover.h" +#include "llvm/Target/TargetOptions.h" + +namespace llvm { + +class Target; + +} namespace lld { namespace elf { @@ -40,14 +48,37 @@ std::vector> compile(); private: + void initLto(BitcodeFile &F); + + void addLto(BitcodeFile &F); + void addThinLto(BitcodeFile &F); + std::vector> runSplitCodegen( const std::function()> &TMFactory); + std::vector> compileLto(); + std::vector> compileThinLto(); + std::unique_ptr compileThinLtoObject(BitcodeFile *F, + SmallString<0> &Obj); + + std::unique_ptr createTargetMachine(); + + // Common + llvm::TargetOptions Options; + std::string TheTriple; + const llvm::Target *TheTarget; + + // Regular LTO + bool HasLto = false; std::unique_ptr Combined; llvm::IRMover Mover; - std::vector> OwningData; + std::vector> Objects; llvm::StringSet<> InternalizedSyms; - std::string TheTriple; + + // ThinLTO + llvm::ModuleSummaryIndex CombinedIndex; + std::vector ThinModules; + std::vector> ThinObjects; }; } } Index: ELF/LTO.cpp =================================================================== --- ELF/LTO.cpp +++ ELF/LTO.cpp @@ -20,8 +20,11 @@ #include "llvm/CodeGen/ParallelCG.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/Linker/IRMover.h" +#include "llvm/Object/ModuleSummaryIndexObjectFile.h" #include "llvm/Support/StringSaver.h" #include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/thread.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" @@ -58,7 +61,8 @@ // Run LTO passes. // Note that the gold plugin has a similar piece of code, so // it is probably better to move this code to a common place. -static void runLTOPasses(Module &M, TargetMachine &TM) { +static void runLTOPasses(Module &M, TargetMachine &TM, + ModuleSummaryIndex *Summary) { legacy::PassManager LtoPasses; LtoPasses.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis())); PassManagerBuilder PMB; @@ -68,6 +72,7 @@ PMB.LoopVectorize = true; PMB.SLPVectorize = true; PMB.OptLevel = Config->LtoO; + PMB.ModuleSummary = Summary; PMB.populateLTOPassManager(LtoPasses); LtoPasses.run(M); @@ -91,6 +96,40 @@ Mover(*Combined) {} void BitcodeCompiler::add(BitcodeFile &F) { + initLto(F); + if (F.ThinLto) + addThinLto(F); + else + addLto(F); +} + +void BitcodeCompiler::initLto(BitcodeFile &F) { + if (!TheTriple.empty()) + return; + + Options = InitTargetOptionsFromCodeGenFlags(); + TheTriple = getBitcodeTargetTriple(F.MB, Driver->Context); + std::string Msg; + TheTarget = TargetRegistry::lookupTarget(TheTriple, Msg); + if (!TheTarget) + fatal("target not found: " + Msg); +} + +std::unique_ptr BitcodeCompiler::createTargetMachine() { + Reloc::Model R = Config->Pic ? Reloc::PIC_ : Reloc::Static; + return std::unique_ptr( + TheTarget->createTargetMachine(TheTriple, "", "", Options, R)); +} + +// This function is called if we know that the combined LTO object will +// provide a definition of a symbol. It undefines the symbol so that the +// definition in the combined LTO object will replace it when parsed. +static void undefine(Symbol *S) { + replaceBody(S, S->body()->getName(), STV_DEFAULT, 0); +} + +void BitcodeCompiler::addLto(BitcodeFile &F) { + HasLto = true; std::unique_ptr Obj = std::move(F.Obj); std::vector Keep; unsigned BodyIndex = 0; @@ -106,13 +145,6 @@ SmallPtrSet Used; collectUsedGlobalVariables(M, Used, /* CompilerUsed */ false); - // This function is called if we know that the combined LTO object will - // provide a definition of a symbol. It undefines the symbol so that the - // definition in the combined LTO object will replace it when parsed. - auto Undefine = [](Symbol *S) { - replaceBody(S, S->body()->getName(), STV_DEFAULT, 0); - }; - for (const BasicSymbolRef &Sym : Obj->symbols()) { uint32_t Flags = Sym.getFlags(); GlobalValue *GV = Obj->getSymbolGV(Sym.getRawDataRefImpl()); @@ -125,7 +157,7 @@ continue; if (!GV) { // Module asm symbol. - Undefine(S); + undefine(S); continue; } auto *B = dyn_cast(S->body()); @@ -150,7 +182,7 @@ if (shouldInternalize(Used, S, GV)) InternalizedSyms.insert(GV->getName()); else - Undefine(S); + undefine(S); Keep.push_back(GV); } @@ -159,6 +191,31 @@ [](GlobalValue &, IRMover::ValueAdder) {}); } +void BitcodeCompiler::addThinLto(BitcodeFile &F) { + ThinModules.push_back(&F); + + std::unique_ptr Obj = std::move(F.Obj); + std::unique_ptr SummaryObj = + check(object::ModuleSummaryIndexObjectFile::create(F.MB, diagHandler)); + CombinedIndex.mergeFrom(SummaryObj->takeIndex(), ThinModules.size()); + + unsigned BodyIndex = 0; + ArrayRef Syms = F.getSymbols(); + + for (const BasicSymbolRef &Sym : Obj->symbols()) { + uint32_t Flags = Sym.getFlags(); + if (BitcodeFile::shouldSkip(Flags)) + continue; + Symbol *S = Syms[BodyIndex++]; + if (Flags & BasicSymbolRef::SF_Undefined) + continue; + auto *B = dyn_cast(S->body()); + if (!B || B->File != &F) + continue; + undefine(S); + } +} + static void internalize(GlobalValue &GV) { assert(!GV.hasLocalLinkage() && "Trying to internalize a symbol with local linkage!"); @@ -168,11 +225,11 @@ std::vector> BitcodeCompiler::runSplitCodegen( const std::function()> &TMFactory) { unsigned NumThreads = Config->LtoJobs; - OwningData.resize(NumThreads); + Objects.resize(NumThreads); std::list OSs; std::vector OSPtrs; - for (SmallString<0> &Obj : OwningData) { + for (SmallString<0> &Obj : Objects) { OSs.emplace_back(Obj); OSPtrs.push_back(&OSs.back()); } @@ -180,20 +237,23 @@ splitCodeGen(std::move(Combined), OSPtrs, {}, TMFactory); std::vector> ObjFiles; - for (SmallString<0> &Obj : OwningData) + for (SmallString<0> &Obj : Objects) ObjFiles.push_back(createObjectFile( MemoryBufferRef(Obj, "LLD-INTERNAL-combined-lto-object"))); if (Config->SaveTemps) for (unsigned I = 0; I < NumThreads; ++I) - saveLtoObjectFile(OwningData[I], I, NumThreads > 1); + saveLtoObjectFile(Objects[I], I, NumThreads > 1); return ObjFiles; } // Merge all the bitcode files we have seen, codegen the result // and return the resulting ObjectFile. -std::vector> BitcodeCompiler::compile() { +std::vector> BitcodeCompiler::compileLto() { + if (!HasLto) + return {}; + TheTriple = Combined->getTargetTriple(); for (const auto &Name : InternalizedSyms) { GlobalValue *GV = Combined->getNamedValue(Name.first()); @@ -204,20 +264,49 @@ if (Config->SaveTemps) saveBCFile(*Combined, ".lto.bc"); - std::string Msg; - const Target *T = TargetRegistry::lookupTarget(TheTriple, Msg); - if (!T) - fatal("target not found: " + Msg); - TargetOptions Options = InitTargetOptionsFromCodeGenFlags(); - Reloc::Model R = Config->Pic ? Reloc::PIC_ : Reloc::Static; + std::unique_ptr TM = createTargetMachine(); + runLTOPasses(*Combined, *TM, nullptr); + + return runSplitCodegen([&]() { return createTargetMachine(); }); +} + +std::unique_ptr +BitcodeCompiler::compileThinLtoObject(BitcodeFile *F, SmallString<0> &Obj) { + LLVMContext Context; + std::unique_ptr M = check(parseBitcodeFile(F->MB, Context)); - auto CreateTargetMachine = [&]() { - return std::unique_ptr( - T->createTargetMachine(TheTriple, "", "", Options, R)); - }; + std::unique_ptr TM = createTargetMachine(); + runLTOPasses(*M, *TM, &CombinedIndex); - std::unique_ptr TM = CreateTargetMachine(); - runLTOPasses(*Combined, *TM); + raw_svector_ostream OS(Obj); + legacy::PassManager CGPasses; + if (TM->addPassesToEmitFile(CGPasses, OS, TargetMachine::CGFT_ObjectFile)) + report_fatal_error("Failed to setup codegen"); + CGPasses.run(*M); - return runSplitCodegen(CreateTargetMachine); + return createObjectFile(MemoryBufferRef(Obj, "LLD-INTERNAL-thinlto-object")); +} + +std::vector> BitcodeCompiler::compileThinLto() { + llvm::ThreadPool Pool(Config->Threads ? std::thread::hardware_concurrency() + : 1); + std::vector> Result(ThinModules.size()); + ThinObjects.resize(ThinModules.size()); + for (unsigned I = 0, E = ThinModules.size(); I != E; ++I) { + Pool.async( + [&](unsigned I) { + Result[I] = compileThinLtoObject(ThinModules[I], ThinObjects[I]); + }, + I); + } + return Result; +} + +// Combine compilation from regular LTO and ThinLTO. +std::vector> BitcodeCompiler::compile() { + auto Objs = compileLto(); + auto ThinLtoObjs = compileThinLto(); + for (auto &Obj : ThinLtoObjs) + Objs.push_back(std::move(Obj)); + return Objs; } Index: ELF/SymbolTable.h =================================================================== --- ELF/SymbolTable.h +++ ELF/SymbolTable.h @@ -11,11 +11,11 @@ #define LLD_ELF_SYMBOL_TABLE_H #include "InputFiles.h" -#include "LTO.h" #include "llvm/ADT/DenseMap.h" namespace lld { namespace elf { +class BitcodeCompiler; class Lazy; template class OutputSectionBase; struct Symbol; @@ -39,6 +39,9 @@ typedef typename ELFT::uint uintX_t; public: + SymbolTable(); + ~SymbolTable(); + void addFile(std::unique_ptr File); void addCombinedLtoObject(); Index: ELF/SymbolTable.cpp =================================================================== --- ELF/SymbolTable.cpp +++ ELF/SymbolTable.cpp @@ -17,6 +17,7 @@ #include "SymbolTable.h" #include "Config.h" #include "Error.h" +#include "LTO.h" #include "Symbols.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/Support/StringSaver.h" @@ -54,6 +55,12 @@ return F->getName(); } +template +SymbolTable::SymbolTable() {} + +template +SymbolTable::~SymbolTable() {} + // Add symbols in File to the symbol table. template void SymbolTable::addFile(std::unique_ptr File) { @@ -231,6 +238,13 @@ /*File*/ nullptr); } +static bool isRegularLtoInputFile(InputFile *File) { + auto *BC = dyn_cast(File); + if (!BC) + return false; + return !BC->ThinLto; +} + template Symbol *SymbolTable::addUndefined(StringRef Name, uint8_t Binding, uint8_t StOther, uint8_t Type, @@ -239,7 +253,7 @@ bool WasInserted; std::tie(S, WasInserted) = insert(Name, Type, StOther & 3, /*CanOmitFromDynSym*/ false, - /*IsUsedInRegularObj*/ !File || !isa(File), File); + /*IsUsedInRegularObj*/ !File || !isRegularLtoInputFile(File), File); if (WasInserted) { S->Binding = Binding; replaceBody(S, Name, StOther, Type); @@ -411,8 +425,9 @@ bool CanOmitFromDynSym, BitcodeFile *F) { Symbol *S; bool WasInserted; - std::tie(S, WasInserted) = insert(Name, Type, StOther & 3, CanOmitFromDynSym, - /*IsUsedInRegularObj*/ false, F); + std::tie(S, WasInserted) = + insert(Name, Type, StOther & 3, CanOmitFromDynSym, + /*IsUsedInRegularObj*/ !isRegularLtoInputFile(F), F); int Cmp = compareDefinedNonCommon(S, WasInserted, IsWeak ? STB_WEAK : STB_GLOBAL); if (Cmp > 0) Index: test/ELF/lto/Inputs/thinlto.ll =================================================================== --- /dev/null +++ test/ELF/lto/Inputs/thinlto.ll @@ -0,0 +1,6 @@ +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo() { + ret void +} Index: test/ELF/lto/thinlto.ll =================================================================== --- /dev/null +++ test/ELF/lto/thinlto.ll @@ -0,0 +1,38 @@ +; RUN: llvm-as -o %t1.o %s +; RUN: llvm-as -o %t2.o %p/Inputs/thinlto.ll + +; RUN: opt -o %t1-thin.o -module-summary %s +; RUN: opt -o %t2-thin.o -module-summary %p/Inputs/thinlto.ll + +; Functions cannot be imported across the LTO/ThinLTO boundary. + +; RUN: ld.lld -m elf_x86_64 -o %t-thin1 %t1-thin.o %t2.o +; RUN: llvm-nm %t-thin1 | FileCheck --check-prefix=CHECK-NOIMPORT-SYM %s +; RUN: llvm-objdump -d %t-thin1 | FileCheck --check-prefix=CHECK-NOIMPORT-OBJ %s + +; RUN: ld.lld -m elf_x86_64 -o %t-thin2 %t1.o %t2-thin.o +; RUN: llvm-nm %t-thin2 | FileCheck --check-prefix=CHECK-NOIMPORT-SYM %s +; RUN: llvm-objdump -d %t-thin2 | FileCheck --check-prefix=CHECK-NOIMPORT-OBJ %s + +; CHECK-NOIMPORT-SYM-DAG: _start +; CHECK-NOIMPORT-SYM-DAG: foo + +; CHECK-NOIMPORT-OBJ: callq + +; foo() should have been imported into t1. + +; RUN: ld.lld -m elf_x86_64 -o %t %t1-thin.o %t2-thin.o +; RUN: llvm-objdump -d %t | FileCheck --check-prefix=CHECK-IMPORT-OBJ %s + +; CHECK-IMPORT-OBJ: _start: +; CHECK-IMPORT-OBJ-NEXT: retq + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @foo() + +define void @_start() { + call void @foo() + ret void +}