Index: include/llvm/LTO/LTO.h =================================================================== --- /dev/null +++ include/llvm/LTO/LTO.h @@ -0,0 +1,147 @@ +#ifndef LLVM_LTO_LTO_H +#define LLVM_LTO_LTO_H + +#include "llvm/ADT/StringSet.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/Linker/IRMover.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Target/TargetOptions.h" + +namespace llvm { + +class LTO; +class Target; +class raw_pwrite_stream; + +struct SymbolResolution { + SymbolResolution() + : Prevailing(0), DefinitionInLinkageUnit(0), VisibleToRegularObj(0) {} + /// The linker has chosen this definition of the symbol. + unsigned Prevailing : 1; + + /// The definition of this symbol is unpreemptable at runtime and is known to + /// be in this linkage unit. + unsigned DefinitionInLinkageUnit : 1; + + /// The definition of this symbol is visible outside of the LTO unit. + unsigned VisibleToRegularObj : 1; +}; + +class LTO { +public: + LTO(LLVMContext &Ctx); + + /// You can configure LTO by setting these fields. + unsigned OptLevel = 2; + bool DisableVerify = false; + TargetOptions Options; + Reloc::Model RelocModel = Reloc::PIC_; + + class symbol_iterator; + + /// This range defines the enumeration order of the symbols in the given + /// IRObjectFile. + static iterator_range symbols(object::IRObjectFile *Obj) { + return llvm::make_range(symbol_iterator(Obj->symbol_begin()), + symbol_iterator(Obj->symbol_end())); + } + + class symbol_iterator { + object::basic_symbol_iterator I; + bool shouldSkip(const object::BasicSymbolRef &Sym) { + uint32_t Flags = Sym.getFlags(); + if (!(Flags & object::BasicSymbolRef::SF_Global)) + return true; + if (Flags & object::BasicSymbolRef::SF_FormatSpecific) + return true; + return false; + } + + public: + symbol_iterator(object::basic_symbol_iterator I) : I(I) {} + void operator++() { + const object::SymbolicFile *Obj = I->getObject(); + auto E = Obj->symbol_end(); + do { + ++I; + } while (I != E && shouldSkip(*I)); + } + + const object::BasicSymbolRef &operator*() const { + return *I; + } + const object::BasicSymbolRef *operator->() const { + return &*I; + } + + bool operator!=(const symbol_iterator &Other) const { + return I != Other.I; + } + }; + + /// Add an IR object file to the LTO link, using the provided symbol + /// resolutions, which are expected to be in the enumeration order defined by + /// LTO::symbols(). + void add(std::unique_ptr Obj, + ArrayRef Res, + DiagnosticHandlerFunction DiagHandler); + + typedef std::function CodeGeneratorFn; + + /// This type defines a stream callback. The stream callback passed to + /// codegen() is used to add a native object that is generated on the fly. + /// The callee must set up an output stream to write the native object to, then + /// call CG passing the output stream as a parameter. + /// + /// Stream callbacks must be thread safe. + typedef std::function AddStreamFn; + + /// This type defines a file callback. The file callback passed to codegen() is + /// used to add a native object that already exists as a file on disk, for + /// example in a cache. + /// + /// File callbacks must be thread safe. + typedef std::function AddFileFn; + + /// A module hook may be used by a linker to perform actions during code + /// generation. For example, a linker may use this function to implement + /// -save-temps, or to add its own resolved symbols to the module. If this + /// function returns false, any further processing for that task is aborted. + typedef std::function ModuleHookFn; + + /// This module hook is called between internalization and optimization. + ModuleHookFn PreOptModuleHook; + + /// This module hook is called between optimization and code generation. + ModuleHookFn PostOptModuleHook; + + /// Returns the number of code generation tasks (i.e. native object files) + /// that the client may expect. Each stream or file callback is supplied a + /// unique task identifier between 0 and getNumTasks()-1. + size_t getNumTasks() const; + + /// Generates code. This function calls the supplied AddStream and AddFile + /// functions to add native object files to the link. + bool codegen(AddStreamFn AddStream, AddFileFn AddFile); + + LLVMContext &getContext() const { return Ctx; } + +private: + LLVMContext &Ctx; + + Module CombinedModule; + IRMover Mover; + + std::string TheTriple; + const Target *TheTarget; + struct GlobalResolution { + bool UnnamedAddr = true; + bool Internalize = false; + }; + StringMap GlobalResolutions; +}; + +} + +#endif Index: lib/LTO/CMakeLists.txt =================================================================== --- lib/LTO/CMakeLists.txt +++ lib/LTO/CMakeLists.txt @@ -48,6 +48,7 @@ add_llvm_library(LLVMLTO + LTO.cpp LTOModule.cpp LTOCodeGenerator.cpp UpdateCompilerUsed.cpp Index: lib/LTO/LTO.cpp =================================================================== --- /dev/null +++ lib/LTO/LTO.cpp @@ -0,0 +1,149 @@ +#include "llvm/LTO/LTO.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/CodeGen/Analysis.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Linker/IRMover.h" +#include "llvm/Object/ModuleSummaryIndexObjectFile.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" + +using namespace llvm; +using namespace object; + +static void upgradeLinkage(GlobalValue *GV) { + switch (GV->getLinkage()) { + default: + break; + case GlobalValue::LinkOnceAnyLinkage: + GV->setLinkage(GlobalValue::WeakAnyLinkage); + break; + case GlobalValue::LinkOnceODRLinkage: + GV->setLinkage(GlobalValue::WeakODRLinkage); + break; + } +} + +void LTO::add(std::unique_ptr Obj, + ArrayRef Res, + DiagnosticHandlerFunction DiagHandler) { + Module &M = Obj->getModule(); + + if (TheTriple.empty()) { + TheTriple = M.getTargetTriple(); + std::string Msg; + TheTarget = TargetRegistry::lookupTarget(TheTriple, Msg); + } + + M.materializeMetadata(); + UpgradeDebugInfo(M); + + SmallPtrSet Used; + collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false); + + std::vector Keep; + + for (GlobalVariable &GV : M.globals()) + if (GV.hasAppendingLinkage()) + Keep.push_back(&GV); + + auto ResI = Res.begin(); + for (const BasicSymbolRef &Sym : symbols(Obj.get())) { + assert(ResI != Res.end()); + SymbolResolution Res = *ResI++; + + if (!Res.Prevailing) + continue; + + GlobalValue *GV = Obj->getSymbolGV(Sym.getRawDataRefImpl()); + + Keep.push_back(GV); + upgradeLinkage(GV); + + if (Res.VisibleToRegularObj) + continue; + + SmallString<64> Name; + { + raw_svector_ostream OS(Name); + Sym.printName(OS); + } + auto &GlobalRes = GlobalResolutions[Name]; + GlobalRes.UnnamedAddr &= GV->hasUnnamedAddr(); + if (!Used.count(GV)) + GlobalRes.Internalize = true; + + // FIXME: use proposed extern_local linkage for SR_DefinitionInLinkageUnit. + } + + assert(ResI == Res.end()); + + if (Mover.move(Obj->takeModule(), Keep, + [](GlobalValue &, IRMover::ValueAdder) {})) { + DiagHandler(DiagnosticInfoInlineAsm("could not move")); + return; + } +} + +LTO::LTO(LLVMContext &Ctx) + : Ctx(Ctx), CombinedModule("ld-temp.o", Ctx), Mover(CombinedModule) {} + +size_t LTO::getNumTasks() const { + return 1; +} + +bool LTO::codegen(AddStreamFn AddStream, AddFileFn AddFile) { + for (const auto &R : GlobalResolutions) { + GlobalValue *GV = CombinedModule.getNamedValue(R.first()); + assert(GV); + assert(!GV->hasLocalLinkage() && + "Trying to internalize a symbol with local linkage!"); + GV->setUnnamedAddr(R.second.UnnamedAddr); + if (R.second.Internalize) + GV->setLinkage(GlobalValue::InternalLinkage); + } + + if (PreOptModuleHook && !PreOptModuleHook(0, CombinedModule)) + return false; + + std::unique_ptr TM( + TheTarget->createTargetMachine(TheTriple, "", "", Options, RelocModel)); + + CombinedModule.setDataLayout(TM->createDataLayout()); + + legacy::PassManager passes; + passes.add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); + + PassManagerBuilder PMB; + PMB.LibraryInfo = new TargetLibraryInfoImpl(Triple(TM->getTargetTriple())); + PMB.Inliner = createFunctionInliningPass(); + // Unconditionally verify input since it is not verified before this + // point and has unknown origin. + PMB.VerifyInput = true; + PMB.VerifyOutput = !DisableVerify; + PMB.LoopVectorize = true; + PMB.SLPVectorize = true; + PMB.OptLevel = OptLevel; + PMB.populateLTOPassManager(passes); + passes.run(CombinedModule); + + if (PostOptModuleHook && !PostOptModuleHook(0, CombinedModule)) + return false; + + auto CodeGen = [&](raw_pwrite_stream &OS) { + legacy::PassManager CodeGenPasses; + if (TM->addPassesToEmitFile(CodeGenPasses, OS, + TargetMachine::CGFT_ObjectFile)) + report_fatal_error("Failed to setup codegen"); + CodeGenPasses.run(CombinedModule); + }; + + AddStream(0, CodeGen); + + return false; +} Index: test/tools/gold/X86/common.ll =================================================================== --- test/tools/gold/X86/common.ll +++ test/tools/gold/X86/common.ll @@ -11,7 +11,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck %s --check-prefix=A ; Shared library case, we merge @a as common and keep it for the symbol table. -; A: @a = common global i32 0, align 8 +; A: @a = common global [4 x i8] zeroinitializer, align 8 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=emit-llvm \ @@ -19,7 +19,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck %s --check-prefix=B ; (i16 align 8) + (i8 align 16) = i16 align 16 -; B: @a = common global i16 0, align 16 +; B: @a = common global [2 x i8] zeroinitializer, align 16 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=emit-llvm \ @@ -27,7 +27,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck %s --check-prefix=C ; (i16 align 8) + (i8 align 1) = i16 align 8. -; C: @a = common global i16 0, align 8 +; C: @a = common global [2 x i8] zeroinitializer, align 8 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=emit-llvm \ @@ -35,7 +35,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck --check-prefix=EXEC %s ; All IR case, we internalize a after merging. -; EXEC: @a = internal global i32 0, align 8 +; EXEC: @a = internal global [4 x i8] zeroinitializer, align 8 ; RUN: llc %p/Inputs/common.ll -o %t2native.o -filetype=obj ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ @@ -44,4 +44,4 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck --check-prefix=MIXED %s ; Mixed ELF and IR. We keep ours as common so the linker will finish the merge. -; MIXED: @a = common global i16 0, align 8 +; MIXED: @a = common global [2 x i8] zeroinitializer, align 8 Index: test/tools/gold/X86/emit-llvm.ll =================================================================== --- test/tools/gold/X86/emit-llvm.ll +++ test/tools/gold/X86/emit-llvm.ll @@ -25,7 +25,7 @@ target triple = "x86_64-unknown-linux-gnu" -; CHECK-DAG: @g1 = linkonce_odr constant i32 32 +; CHECK-DAG: @g1 = weak_odr constant i32 32 @g1 = linkonce_odr constant i32 32 ; CHECK-DAG: @g2 = internal can_omit_from_dynsym constant i32 32 @@ -63,8 +63,8 @@ ret void } -; CHECK-DAG: define linkonce_odr void @f5() -; OPT-DAG: define linkonce_odr void @f5() +; CHECK-DAG: define weak_odr void @f5() +; OPT-DAG: define weak_odr void @f5() define linkonce_odr void @f5() { ret void } Index: test/tools/gold/X86/start-lib-common.ll =================================================================== --- test/tools/gold/X86/start-lib-common.ll +++ test/tools/gold/X86/start-lib-common.ll @@ -19,4 +19,4 @@ ; Check that the common symbol is not dropped completely, which was a regression ; in r262676. -; CHECK: @x = common global i32 0 +; CHECK: @x = common global [4 x i8] zeroinitializer Index: tools/gold/CMakeLists.txt =================================================================== --- tools/gold/CMakeLists.txt +++ tools/gold/CMakeLists.txt @@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} + LTO Linker BitWriter IPO Index: tools/gold/gold-plugin.cpp =================================================================== --- tools/gold/gold-plugin.cpp +++ tools/gold/gold-plugin.cpp @@ -28,6 +28,7 @@ #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" +#include "llvm/LTO/LTO.h" #include "llvm/Linker/IRMover.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Object/IRObjectFile.h" @@ -100,14 +101,14 @@ }; struct ResolutionInfo { - uint64_t CommonSize = 0; - unsigned CommonAlign = 0; - bool IsLinkonceOdr = true; - bool UnnamedAddr = true; bool CanOmitFromDynSym = true; - GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility; - bool CommonInternal = false; - bool UseCommon = false; +}; + +struct CommonResolution { + bool Prevailing = false; + bool VisibleToRegularObj = false; + uint64_t Size = 0; + unsigned Align = 0; }; /// Class to own information used by a task or during its cleanup for a @@ -147,6 +148,7 @@ static std::string output_name = ""; static std::list Modules; static StringMap ResInfo; +static std::map Commons; static std::vector Cleanup; static llvm::TargetOptions TargetOpts; static std::string DefaultTriple = sys::getDefaultTargetTriple(); @@ -383,14 +385,6 @@ return cast(&GV); } -static bool shouldSkip(uint32_t Symflags) { - if (!(Symflags & object::BasicSymbolRef::SF_Global)) - return true; - if (Symflags & object::BasicSymbolRef::SF_FormatSpecific) - return true; - return false; -} - static void diagnosticHandler(const DiagnosticInfo &DI) { if (const auto *BDI = dyn_cast(&DI)) { std::error_code EC = BDI->getError(); @@ -425,18 +419,6 @@ diagnosticHandler(DI); } -static GlobalValue::VisibilityTypes -getMinVisibility(GlobalValue::VisibilityTypes A, - GlobalValue::VisibilityTypes B) { - if (A == GlobalValue::HiddenVisibility) - return A; - if (B == GlobalValue::HiddenVisibility) - return B; - if (A == GlobalValue::ProtectedVisibility) - return A; - return B; -} - /// Called by gold to see whether this file is one that our plugin can handle. /// We'll try to open it and register all the symbols with add_symbol if /// possible. @@ -493,15 +475,8 @@ cf.handle = file->handle; - // If we are doing ThinLTO compilation, don't need to process the symbols. - // Later we simply build a combined index file after all files are claimed. - if (options::thinlto && options::thinlto_index_only) - return LDPS_OK; - - for (auto &Sym : Obj->symbols()) { + for (auto &Sym : LTO::symbols(Obj.get())) { uint32_t Symflags = Sym.getFlags(); - if (shouldSkip(Symflags)) - continue; cf.syms.push_back(ld_plugin_symbol()); ld_plugin_symbol &sym = cf.syms.back(); @@ -514,16 +489,14 @@ } sym.name = strdup(Name.c_str()); + ResolutionInfo &Res = ResInfo[Name]; + const GlobalValue *GV = Obj->getSymbolGV(Sym.getRawDataRefImpl()); - ResolutionInfo &Res = ResInfo[sym.name]; + Res.CanOmitFromDynSym &= (GV && GV->canOmitFromDynSym()); sym.visibility = LDPV_DEFAULT; if (GV) { - Res.UnnamedAddr &= GV->hasUnnamedAddr(); - Res.CanOmitFromDynSym &= GV->canOmitFromDynSym(); - Res.IsLinkonceOdr &= GV->hasLinkOnceLinkage(); - Res.Visibility = getMinVisibility(Res.Visibility, GV->getVisibility()); switch (GV->getVisibility()) { case GlobalValue::DefaultVisibility: break; @@ -576,13 +549,6 @@ return LDPS_OK; } -static void internalize(GlobalValue &GV) { - if (GV.isDeclarationForLinker()) - return; // We get here if there is a matching asm definition. - if (!GV.hasLocalLinkage()) - GV.setLinkage(GlobalValue::InternalLinkage); -} - static const char *getResolutionName(ld_plugin_symbol_resolution R) { switch (R) { case LDPR_UNKNOWN: @@ -632,44 +598,12 @@ return View; } -static std::unique_ptr -getModuleSummaryIndexForFile(claimed_file &F, ld_plugin_input_file &Info) { - const void *View = getSymbolsAndView(F); - if (!View) - return nullptr; - - MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize), - Info.name); - - // Don't bother trying to build an index if there is no summary information - // in this bitcode file. - if (!object::ModuleSummaryIndexObjectFile::hasGlobalValueSummaryInMemBuffer( - BufferRef, diagnosticHandler)) - return std::unique_ptr(nullptr); - - ErrorOr> ObjOrErr = - object::ModuleSummaryIndexObjectFile::create(BufferRef, - diagnosticHandler); - - if (std::error_code EC = ObjOrErr.getError()) - message(LDPL_FATAL, - "Could not read module summary index bitcode from file : %s", - EC.message().c_str()); - - object::ModuleSummaryIndexObjectFile &Obj = **ObjOrErr; - - return Obj.takeIndex(); -} - -static std::unique_ptr -getModuleForFile(LLVMContext &Context, claimed_file &F, const void *View, - ld_plugin_input_file &Info, raw_fd_ostream *ApiFile, - StringSet<> &Internalize, std::vector &Keep, - StringMap &Realign) { +static void addModule(LTO &Lto, claimed_file &F, const void *View, + ld_plugin_input_file &Info, raw_fd_ostream *ApiFile) { MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize), Info.name); ErrorOr> ObjOrErr = - object::IRObjectFile::create(BufferRef, Context); + object::IRObjectFile::create(BufferRef, Lto.getContext()); if (std::error_code EC = ObjOrErr.getError()) message(LDPL_FATAL, "Could not read bitcode from file : %s", @@ -678,22 +612,17 @@ object::IRObjectFile &Obj = **ObjOrErr; Module &M = Obj.getModule(); - - M.materializeMetadata(); - UpgradeDebugInfo(M); - - SmallPtrSet Used; - collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false); + if (!options::triple.empty()) + M.setTargetTriple(options::triple.c_str()); + else if (M.getTargetTriple().empty()) + M.setTargetTriple(DefaultTriple); unsigned SymNum = 0; - for (auto &ObjSym : Obj.symbols()) { + std::vector Resols(F.syms.size()); + for (auto &ObjSym : LTO::symbols(&Obj)) { GlobalValue *GV = Obj.getSymbolGV(ObjSym.getRawDataRefImpl()); - if (GV && GV->hasAppendingLinkage()) - Keep.push_back(GV); - - if (shouldSkip(ObjSym.getFlags())) - continue; ld_plugin_symbol &Sym = F.syms[SymNum]; + SymbolResolution &R = Resols[SymNum]; ++SymNum; ld_plugin_symbol_resolution Resolution = @@ -702,62 +631,7 @@ if (options::generate_api_file) *ApiFile << Sym.name << ' ' << getResolutionName(Resolution) << '\n'; - if (!GV) { - freeSymName(Sym); - continue; // Asm symbol. - } - ResolutionInfo &Res = ResInfo[Sym.name]; - if (Resolution == LDPR_PREVAILING_DEF_IRONLY_EXP && !Res.IsLinkonceOdr) - Resolution = LDPR_PREVAILING_DEF; - - // In ThinLTO mode change all prevailing resolutions to LDPR_PREVAILING_DEF. - // For ThinLTO the IR files are compiled through the backend independently, - // so we need to ensure that any prevailing linkonce copy will be emitted - // into the object file by making it weak. Additionally, we can skip the - // IRONLY handling for internalization, which isn't performed in ThinLTO - // mode currently anyway. - if (options::thinlto && (Resolution == LDPR_PREVAILING_DEF_IRONLY_EXP || - Resolution == LDPR_PREVAILING_DEF_IRONLY)) - Resolution = LDPR_PREVAILING_DEF; - - GV->setUnnamedAddr(Res.UnnamedAddr); - GV->setVisibility(Res.Visibility); - - // Override gold's resolution for common symbols. We want the largest - // one to win. - if (GV->hasCommonLinkage()) { - if (Resolution == LDPR_PREVAILING_DEF_IRONLY) - Res.CommonInternal = true; - - if (Resolution == LDPR_PREVAILING_DEF_IRONLY || - Resolution == LDPR_PREVAILING_DEF) - Res.UseCommon = true; - - const DataLayout &DL = GV->getParent()->getDataLayout(); - uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType()); - unsigned Align = GV->getAlignment(); - - if (Res.UseCommon && Size >= Res.CommonSize) { - // Take GV. - if (Res.CommonInternal) - Resolution = LDPR_PREVAILING_DEF_IRONLY; - else - Resolution = LDPR_PREVAILING_DEF; - cast(GV)->setAlignment( - std::max(Res.CommonAlign, Align)); - } else { - // Do not take GV, it's smaller than what we already have in the - // combined module. - Resolution = LDPR_PREEMPTED_IR; - if (Align > Res.CommonAlign) - // Need to raise the alignment though. - Realign[Sym.name] = Align; - } - - Res.CommonSize = std::max(Res.CommonSize, Size); - Res.CommonAlign = std::max(Res.CommonAlign, Align); - } switch (Resolution) { case LDPR_UNKNOWN: @@ -765,52 +639,56 @@ case LDPR_RESOLVED_IR: case LDPR_RESOLVED_EXEC: - case LDPR_RESOLVED_DYN: case LDPR_PREEMPTED_IR: case LDPR_PREEMPTED_REG: + // FIXME: Not quite right, need to take into account run-time preemption. + R.DefinitionInLinkageUnit = true; break; + case LDPR_RESOLVED_DYN: case LDPR_UNDEF: - if (!GV->isDeclarationForLinker()) - assert(GV->hasComdat()); break; - case LDPR_PREVAILING_DEF_IRONLY: { - Keep.push_back(GV); - // The IR linker has to be able to map this value to a declaration, - // so we can only internalize after linking. - if (!Used.count(GV)) - Internalize.insert(GV->getName()); + case LDPR_PREVAILING_DEF_IRONLY: + R.Prevailing = true; break; - } case LDPR_PREVAILING_DEF: - Keep.push_back(GV); - // There is a non IR use, so we have to force optimizations to keep this. - switch (GV->getLinkage()) { - default: - break; - case GlobalValue::LinkOnceAnyLinkage: - GV->setLinkage(GlobalValue::WeakAnyLinkage); - break; - case GlobalValue::LinkOnceODRLinkage: - GV->setLinkage(GlobalValue::WeakODRLinkage); - break; - } + R.Prevailing = true; + R.VisibleToRegularObj = true; break; - case LDPR_PREVAILING_DEF_IRONLY_EXP: { - Keep.push_back(GV); - if (Res.CanOmitFromDynSym) - Internalize.insert(GV->getName()); + case LDPR_PREVAILING_DEF_IRONLY_EXP: + R.Prevailing = true; + if (!Res.CanOmitFromDynSym) + R.VisibleToRegularObj = true; break; } + + if (GV && GV->hasCommonLinkage()) { + // We ignore gold's resolution for common symbols. A common symbol with + // the correct size and alignment is added to the module by the pre-opt + // module hook if any common symbol prevailed. + CommonResolution &CommonRes = Commons[GV->getName()]; + if (R.Prevailing) { + CommonRes.Prevailing = true; + CommonRes.VisibleToRegularObj = R.VisibleToRegularObj; + } + const DataLayout &DL = GV->getParent()->getDataLayout(); + CommonRes.Size = std::max( + CommonRes.Size, DL.getTypeAllocSize(GV->getType()->getElementType())); + CommonRes.Align = std::max(CommonRes.Align, GV->getAlignment()); + R.Prevailing = false; + // If we've seen a common, the linker should either pick it or some other + // symbol in the same linkage unit. + // FIXME: Not quite right, need to take into account run-time preemption. + R.DefinitionInLinkageUnit = true; } freeSymName(Sym); } - return Obj.takeModule(); + Lto.add(std::move(*ObjOrErr), Resols, diagnosticHandler); } static void saveBCFile(StringRef Path, Module &M) { @@ -1081,207 +959,12 @@ /// Links the module in \p View from file \p F into the combined module /// saved in the IRMover \p L. Returns true on error, false on success. -static bool linkInModule(LLVMContext &Context, IRMover &L, claimed_file &F, - const void *View, ld_plugin_input_file &File, - raw_fd_ostream *ApiFile, StringSet<> &Internalize) { - std::vector Keep; - StringMap Realign; - std::unique_ptr M = getModuleForFile( - Context, F, View, File, ApiFile, Internalize, Keep, Realign); - if (!M.get()) - return false; - if (!options::triple.empty()) - M->setTargetTriple(options::triple.c_str()); - else if (M->getTargetTriple().empty()) { - M->setTargetTriple(DefaultTriple); - } - - if (L.move(std::move(M), Keep, [](GlobalValue &, IRMover::ValueAdder) {})) - return true; - - for (const auto &I : Realign) { - GlobalValue *Dst = L.getModule().getNamedValue(I.first()); - if (!Dst) - continue; - cast(Dst)->setAlignment(I.second); - } - +static bool linkInModule(LTO &Lto, claimed_file &F, const void *View, + ld_plugin_input_file &File, raw_fd_ostream *ApiFile) { + addModule(Lto, F, View, File, ApiFile); return false; } -/// Perform the ThinLTO backend on a single module, invoking the LTO and codegen -/// pipelines. -static void thinLTOBackendTask(claimed_file &F, const void *View, - ld_plugin_input_file &File, - raw_fd_ostream *ApiFile, - const ModuleSummaryIndex &CombinedIndex, - raw_fd_ostream *OS, unsigned TaskID) { - // Need to use a separate context for each task - LLVMContext Context; - Context.setDiscardValueNames(options::TheOutputType != - options::OT_SAVE_TEMPS); - Context.enableDebugTypeODRUniquing(); // Merge debug info types. - Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true); - - std::unique_ptr NewModule(new llvm::Module(File.name, Context)); - IRMover L(*NewModule.get()); - - StringSet<> Dummy; - if (linkInModule(Context, L, F, View, File, ApiFile, Dummy)) - message(LDPL_FATAL, "Failed to rename module for ThinLTO"); - if (renameModuleForThinLTO(*NewModule, CombinedIndex)) - message(LDPL_FATAL, "Failed to rename module for ThinLTO"); - - CodeGen codeGen(std::move(NewModule), OS, TaskID, &CombinedIndex, File.name); - codeGen.runAll(); -} - -/// Launch each module's backend pipeline in a separate task in a thread pool. -static void thinLTOBackends(raw_fd_ostream *ApiFile, - const ModuleSummaryIndex &CombinedIndex) { - unsigned TaskCount = 0; - std::vector Tasks; - Tasks.reserve(Modules.size()); - unsigned int MaxThreads = options::Parallelism - ? options::Parallelism - : thread::hardware_concurrency(); - - // Create ThreadPool in nested scope so that threads will be joined - // on destruction. - { - ThreadPool ThinLTOThreadPool(MaxThreads); - for (claimed_file &F : Modules) { - // Do all the gold callbacks in the main thread, since gold is not thread - // safe by default. - PluginInputFile InputFile(F.handle); - const void *View = getSymbolsAndView(F); - if (!View) - continue; - - SmallString<128> Filename; - if (!options::obj_path.empty()) - // Note that openOutputFile will append a unique ID for each task - Filename = options::obj_path; - else if (options::TheOutputType == options::OT_SAVE_TEMPS) { - // Use the input file name so that we get a unique and identifiable - // output file for each ThinLTO backend task. - Filename = InputFile.file().name; - Filename += ".thinlto.o"; - } - bool TempOutFile = Filename.empty(); - - SmallString<128> NewFilename; - int FD = openOutputFile(Filename, TempOutFile, NewFilename, - // Only append the TaskID if we will use the - // non-unique obj_path. - !options::obj_path.empty() ? TaskCount : -1); - TaskCount++; - std::unique_ptr OS = - llvm::make_unique(FD, true); - - // Enqueue the task - ThinLTOThreadPool.async(thinLTOBackendTask, std::ref(F), View, - std::ref(InputFile.file()), ApiFile, - std::ref(CombinedIndex), OS.get(), TaskCount); - - // Record the information needed by the task or during its cleanup - // to a ThinLTOTaskInfo instance. For information needed by the task - // the unique_ptr ownership is transferred to the ThinLTOTaskInfo. - Tasks.emplace_back(std::move(InputFile), std::move(OS), - NewFilename.c_str(), TempOutFile); - } - } - - for (auto &Task : Tasks) - Task.cleanup(); -} - -/// Perform ThinLTO link, which creates the combined index file. -/// Also, either launch backend threads or (under thinlto-index-only) -/// emit individual index files for distributed backends and exit. -static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) { - ModuleSummaryIndex CombinedIndex; - uint64_t NextModuleId = 0; - for (claimed_file &F : Modules) { - PluginInputFile InputFile(F.handle); - - std::unique_ptr Index = - getModuleSummaryIndexForFile(F, InputFile.file()); - - // Skip files without a module summary. - if (Index) - CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId); - } - - if (options::thinlto_emit_imports_files && !options::thinlto_index_only) - message(LDPL_WARNING, - "thinlto-emit-imports-files ignored unless thinlto-index-only"); - - if (options::thinlto_index_only) { - // Collect for each module the list of function it defines (GUID -> - // Summary). - StringMap> - ModuleToDefinedGVSummaries(NextModuleId); - CombinedIndex.collectDefinedGVSummariesPerModule( - ModuleToDefinedGVSummaries); - - // FIXME: We want to do this for the case where the threads are launched - // from gold as well, in which case this will be moved out of the - // thinlto_index_only handling, and the function importer will be invoked - // directly using the Lists. - StringMap ImportLists(NextModuleId); - StringMap ExportLists(NextModuleId); - ComputeCrossModuleImport(CombinedIndex, ModuleToDefinedGVSummaries, - ImportLists, ExportLists); - - // For each input bitcode file, generate an individual index that - // contains summaries only for its own global values, and for any that - // should be imported. - for (claimed_file &F : Modules) { - PluginInputFile InputFile(F.handle); - std::error_code EC; - raw_fd_ostream OS((Twine(InputFile.file().name) + ".thinlto.bc").str(), - EC, sys::fs::OpenFlags::F_None); - if (EC) - message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", - InputFile.file().name, EC.message().c_str()); - // Build a map of module to the GUIDs and summary objects that should - // be written to its index. - std::map ModuleToSummariesForIndex; - gatherImportedSummariesForModule(InputFile.file().name, - ModuleToDefinedGVSummaries, ImportLists, - ModuleToSummariesForIndex); - WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); - - if (options::thinlto_emit_imports_files) { - if ((EC = EmitImportsFiles( - InputFile.file().name, - (Twine(InputFile.file().name) + ".imports").str(), - ImportLists))) - message(LDPL_FATAL, "Unable to open %s.imports", - InputFile.file().name, EC.message().c_str()); - } - } - - cleanup_hook(); - exit(0); - } - - // Create OS in nested scope so that it will be closed on destruction. - { - std::error_code EC; - raw_fd_ostream OS(output_name + ".thinlto.bc", EC, - sys::fs::OpenFlags::F_None); - if (EC) - message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", - output_name.data(), EC.message().c_str()); - WriteIndexToFile(CombinedIndex, OS); - } - - thinLTOBackends(ApiFile, CombinedIndex); - return LDPS_OK; -} - /// gold informs us that all symbols have been read. At this point, we use /// get_symbols to see if any of our definitions have been overridden by a /// native object file. Then, perform optimization and codegen. @@ -1292,51 +975,109 @@ if (unsigned NumOpts = options::extra.size()) cl::ParseCommandLineOptions(NumOpts, &options::extra[0]); - if (options::thinlto) - return thinLTOLink(ApiFile); - LLVMContext Context; Context.setDiscardValueNames(options::TheOutputType != options::OT_SAVE_TEMPS); Context.enableDebugTypeODRUniquing(); // Merge debug info types. Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true); - std::unique_ptr Combined(new Module("ld-temp.o", Context)); - IRMover L(*Combined); + LTO Lto(Context); + Lto.DisableVerify = options::DisableVerify; + Lto.OptLevel = options::OptLevel; - StringSet<> Internalize; for (claimed_file &F : Modules) { PluginInputFile InputFile(F.handle); const void *View = getSymbolsAndView(F); if (!View) continue; - if (linkInModule(Context, L, F, View, InputFile.file(), ApiFile, - Internalize)) + if (linkInModule(Lto, F, View, InputFile.file(), ApiFile)) message(LDPL_FATAL, "Failed to link module"); } - for (const auto &Name : Internalize) { - GlobalValue *GV = Combined->getNamedValue(Name.first()); - if (GV) - internalize(*GV); - } + SmallString<128> Filename; + // Note that openOutputFile will append a unique ID for each task + if (!options::obj_path.empty()) + Filename = options::obj_path; + else if (options::TheOutputType == options::OT_SAVE_TEMPS) + Filename = output_name + ".o"; + bool SaveTemps = !Filename.empty(); + + size_t NumTasks = Lto.getNumTasks(); + std::vector IsTemporary(NumTasks); + std::vector> Filenames(NumTasks); + + Lto.PreOptModuleHook = [&](size_t Task, Module &M) -> bool { + if (Task == 0) + for (auto &I : Commons) { + if (!I.second.Prevailing) + continue; + ArrayType *Ty = + ArrayType::get(Type::getInt8Ty(M.getContext()), I.second.Size); + GlobalVariable *OldGV = M.getNamedGlobal(I.first); + auto *GV = new GlobalVariable(M, Ty, false, GlobalValue::CommonLinkage, + ConstantAggregateZero::get(Ty), ""); + GV->setAlignment(I.second.Align); + if (OldGV) { + OldGV->replaceAllUsesWith( + ConstantExpr::getBitCast(GV, OldGV->getType())); + GV->takeName(OldGV); + OldGV->eraseFromParent(); + } else { + GV->setName(I.first); + } + if (!I.second.VisibleToRegularObj) + GV->setLinkage(GlobalValue::InternalLinkage); + } - if (options::TheOutputType == options::OT_DISABLE) - return LDPS_OK; + if (options::TheOutputType == options::OT_DISABLE) + return false; + + if (options::TheOutputType != options::OT_NORMAL) { + std::string path; + if (options::TheOutputType == options::OT_BC_ONLY) + path = output_name; + else + path = output_name + ".bc"; + saveBCFile(path, M); + if (options::TheOutputType == options::OT_BC_ONLY) + return false; + } - if (options::TheOutputType != options::OT_NORMAL) { - std::string path; - if (options::TheOutputType == options::OT_BC_ONLY) - path = output_name; - else - path = output_name + ".bc"; - saveBCFile(path, *Combined); - if (options::TheOutputType == options::OT_BC_ONLY) - return LDPS_OK; - } + return true; + }; + + Lto.PostOptModuleHook = [&](size_t Task, Module &M) -> bool { + SmallString<128> OptFilename; + if (options::TheOutputType == options::OT_SAVE_TEMPS) { + OptFilename = output_name + ".opt.bc"; + saveBCFile(OptFilename, M); + } + + return true; + }; + + auto AddStream = [&](size_t Task, LTO::CodeGeneratorFn CodeGen) { + int FD = openOutputFile(Filename, /*TempOutFile=*/!SaveTemps, + Filenames[Task], NumTasks > 1 ? Task : -1); + IsTemporary[Task] = !SaveTemps; + + llvm::raw_fd_ostream OS(FD, true); + CodeGen(OS); + }; + + auto AddFile = [&](size_t Task, StringRef S) { + Filenames[Task] = S; + IsTemporary[Task] = false; + }; + + Lto.codegen(AddStream, AddFile); + + if (options::TheOutputType == options::OT_DISABLE || + options::TheOutputType == options::OT_BC_ONLY) + return LDPS_OK; - CodeGen codeGen(std::move(Combined)); - codeGen.runAll(); + for (unsigned I = 0; I != NumTasks; ++I) + recordFile(Filenames[I].str(), IsTemporary[I]); if (!options::extra_library_path.empty() && set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK)