Index: include/llvm/CodeGen/LTOBackend.h =================================================================== --- /dev/null +++ include/llvm/CodeGen/LTOBackend.h @@ -0,0 +1,138 @@ +#ifndef LLVM_TRANSFORMS_IPO_LTOBACKEND_H +#define LLVM_TRANSFORMS_IPO_LTOBACKEND_H + +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO/FunctionImport.h" + +#include + +namespace llvm { + +class Module; +class Target; +class raw_pwrite_stream; + +namespace lto { + +struct Config { + std::string CPU; + std::string Features; + TargetOptions Options; + std::vector MAttrs; + Reloc::Model RelocModel = Reloc::PIC_; + CodeModel::Model CodeModel = CodeModel::Default; + CodeGenOpt::Level CGOptLevel = CodeGenOpt::Default; + unsigned OptLevel = 2; + bool DisableVerify = false; + + /// Setting this field will replace target triples in input files with this + /// triple. + std::string OverrideTriple; + + /// Setting this field will replace unspecified target triples in input files + /// with this triple. + std::string DefaultTriple; + + bool ShouldDiscardValueNames = true; + DiagnosticHandlerFunction DiagHandler; + + /// The following callbacks deal with tasks, which normally represent the + /// entire optimization and code generation pipeline for what will become a + /// single native object file. Each task has a unique identifier between 0 and + /// getMaxTasks()-1, which is supplied to the callback via the Task parameter. + /// A task represents the entire pipeline for ThinLTO and regular + /// (non-parallel) LTO, but a parallel code generation task will be split into + /// N tasks before code generation, where N is the parallelism level. + /// + /// LTO may decide to stop processing a task at any time, for example if the + /// module is empty or if a module hook (see below) returns false. For this + /// reason, the client should not expect to receive exactly getMaxTasks() + /// native object files. + + /// A module hook may be used by a linker to perform actions during the LTO + /// pipeline. For example, a linker may use this function to implement + /// -save-temps, or to add its own resolved symbols to the module. If this + /// function returns false, any further processing for that task is aborted. + /// + /// Module hooks must be thread safe. + typedef std::function ModuleHookFn; + + /// This module hook is called after linking (regular LTO) or loading + /// (ThinLTO) the module, before modifying it. + ModuleHookFn PreOptModuleHook; + + /// This hook is called after promoting any internal functions + /// (ThinLTO-specific). + ModuleHookFn PostPromoteModuleHook; + + /// This hook is called after internalizing the module. + ModuleHookFn PostInternalizeModuleHook; + + /// This hook is called after importing from other modules (ThinLTO-specific). + ModuleHookFn PostImportModuleHook; + + /// This module hook is called after optimization is complete. + ModuleHookFn PostOptModuleHook; + + /// This module hook is called before code generation. It is similar to the + /// PostOptModuleHook, but for parallel code generation it is called after + /// splitting the module. + ModuleHookFn PreCodeGenModuleHook; + + /// A combined index hook is called after all per-module indexes have been + /// combined (ThinLTO-specific). It can be used to implement -save-temps for + /// the combined index. + /// + /// If this function returns false, any further processing for ThinLTO tasks + /// is aborted. + typedef std::function CombinedIndexHookFn; + CombinedIndexHookFn CombinedIndexHook; +}; + +/// This type defines a stream callback. A stream callback is used to add a +/// native object that is generated on the fly. The callee must set up and +/// return a output stream to write the native object to. +/// +/// Stream callbacks must be thread safe. +typedef std::function(size_t Task)> + AddStreamFn; + +/// Runs a regular LTO backend. +bool backend(Config &C, StringRef TheTriple, const Target *TheTarget, + AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, + std::unique_ptr M); + +/// Runs a ThinLTO backend. +bool thinBackend(Config &C, StringRef TheTriple, const Target *TheTarget, + size_t Task, AddStreamFn AddStream, Module &M, + ModuleSummaryIndex &CombinedIndex, + const FunctionImporter::ImportMapTy &ImportList, + StringMap &ModuleMap); + +/// If GV's linkage is linkonce, change it to weak so that it won't be DCE'd. +void upgradeLinkage(GlobalValue *GV); + +/// A derived class of LLVMContext that initializes itself according to a given +/// Config object. +struct LTOLLVMContext : LLVMContext { + static void funcDiagHandler(const DiagnosticInfo &DI, void *Context) { + auto *Fn = + static_cast *>(Context); + (*Fn)(DI); + } + + LTOLLVMContext(const Config &C) : DiagHandler(C.DiagHandler) { + setDiscardValueNames(C.ShouldDiscardValueNames); + enableDebugTypeODRUniquing(); + setDiagnosticHandler(funcDiagHandler, &DiagHandler, true); + } + std::function DiagHandler; +}; + +} +} + +#endif Index: include/llvm/LTO/LTO.h =================================================================== --- /dev/null +++ include/llvm/LTO/LTO.h @@ -0,0 +1,333 @@ +#ifndef LLVM_LTO_LTO_H +#define LLVM_LTO_LTO_H + +#include "llvm/ADT/StringSet.h" +#include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/LTOBackend.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/Linker/IRMover.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Support/thread.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO/FunctionImport.h" + +namespace llvm { + +class Error; +class Target; +class raw_pwrite_stream; + +namespace lto { + +enum class lto_error { + comdat_alias = 1, + unable_to_find_target, +}; + +const std::error_category <o_category(); + +class LTO; +struct SymbolResolution; +class ThinBackendProc; + +/// An input file. This is a wrapper for IRObjectFile that exposes only the +/// information that an LTO client should need in order to do symbol resolution. +class InputFile { + friend LTO; + InputFile(std::unique_ptr Obj) : Obj(std::move(Obj)) {} + + std::unique_ptr Obj; + +public: + /// Create an InputFile. FIXME: Remove Lto parameter here. It is currently + /// needed to supply an owning LLVMContext for the IRObjectFile. + static ErrorOr> create(MemoryBufferRef Object, + LTO &Lto); + + class symbol_iterator; + + /// This is a wrapper for object::basic_symbol_iterator that exposes only the + /// information that an LTO client should need in order to do symbol + /// resolution. + /// + /// This object is ephemeral; it is only valid as long as an iterator obtained + /// from symbols() refers to it. + class Symbol { + friend symbol_iterator; + friend LTO; + + object::basic_symbol_iterator I; + const GlobalValue *GV; + uint32_t Flags; + SmallString<64> Name; + + bool shouldSkip() { + if (!(Flags & object::BasicSymbolRef::SF_Global)) + return true; + return Flags & object::BasicSymbolRef::SF_FormatSpecific; + } + + void skip() { + const object::SymbolicFile *Obj = I->getObject(); + auto E = Obj->symbol_end(); + while (I != E) { + Flags = I->getFlags(); + if (!shouldSkip()) + break; + ++I; + } + if (I == E) + return; + + Name.clear(); + { + raw_svector_ostream OS(Name); + I->printName(OS); + } + GV = cast(Obj)->getSymbolGV(I->getRawDataRefImpl()); + } + + public: + Symbol(object::basic_symbol_iterator I) : I(I) { skip(); } + + StringRef getName() const { return Name; } + StringRef getIRName() const { + if (GV) + return GV->getName(); + return StringRef(); + } + uint32_t getFlags() const { return Flags; } + GlobalValue::VisibilityTypes getVisibility() const { + if (GV) + return GV->getVisibility(); + return GlobalValue::DefaultVisibility; + } + bool canBeOmittedFromSymbolTable() const { + return GV && llvm::canBeOmittedFromSymbolTable(GV); + } + Expected getComdat() const { + const GlobalObject *GO; + if (auto *GA = dyn_cast(GV)) { + GO = GA->getBaseObject(); + if (!GO) + return errorCodeToError(std::error_code( + static_cast(lto_error::comdat_alias), lto_category())); + } else { + GO = cast(GV); + } + if (GV) + return GV->getComdat(); + return nullptr; + } + size_t getCommonSize() const { + assert(Flags & object::BasicSymbolRef::SF_Common); + if (!GV) + return 0; + return GV->getParent()->getDataLayout().getTypeAllocSize( + GV->getType()->getElementType()); + } + unsigned getCommonAlignment() const { + assert(Flags & object::BasicSymbolRef::SF_Common); + if (!GV) + return 0; + return GV->getAlignment(); + } + }; + + class symbol_iterator { + Symbol Sym; + + public: + symbol_iterator(object::basic_symbol_iterator I) : Sym(I) {} + + symbol_iterator &operator++() { + ++Sym.I; + Sym.skip(); + return *this; + } + + symbol_iterator operator++(int) { + symbol_iterator I = *this; + ++*this; + return I; + } + + const Symbol &operator*() const { return Sym; } + const Symbol *operator->() const { return &Sym; } + + bool operator!=(const symbol_iterator &Other) const { + return Sym.I != Other.Sym.I; + } + }; + + /// A range over the symbols in this InputFile. + iterator_range symbols() { + return llvm::make_range(symbol_iterator(Obj->symbol_begin()), + symbol_iterator(Obj->symbol_end())); + } +}; + +/// A ThinBackend defines what happens after the thin-link phase during ThinLTO. +/// The details of this type definition aren't important; clients can only +/// create a ThinBackend using one of the create*ThinBackend() functions below. +typedef std::function( + Config &C, StringRef TheTriple, const Target *TheTarget, + ModuleSummaryIndex &CombinedIndex, AddStreamFn AddStream)> + ThinBackend; + +/// This ThinBackend runs the individual backend jobs in-process. +ThinBackend createInProcessThinBackend(unsigned ParallelismLevel); + +/// This ThinBackend writes individual module indexes to files, instead of +/// running the individual backend jobs. This backend is for distributed builds +/// where separate processes will invoke the real backends. +/// +/// To find the path to write the index to, the backend checks if the path has a +/// prefix of OldPrefix; if so, it replaces that prefix with NewPrefix. It then +/// appends ".thinlto.bc" and writes the index to that path. If +/// ShouldEmitImportsFiles is true it also writes a list of imported files to a +/// similar path with ".imports" appended instead. +ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, + std::string NewPrefix, + bool ShouldEmitImportsFiles); + +/// This class implements a resolution-based interface to LLVM's LTO +/// functionality. It supports regular LTO, parallel LTO code generation and +/// ThinLTO. You can use it from a linker in the following way: +/// - Set hooks and code generation options (see lto::Config struct defined in +/// lib/CodeGen/LTOBackend.h), and use the lto::Config object to create an +/// lto::LTO object. +/// - Create lto::InputFile objects using lto::InputFile::create(), then use +/// the symbols() function to enumerate its symbols and compute a resolution +/// for each symbol (see SymbolResolution below). +/// - After the linker has visited each input file (and each regular object +/// file) and computed a resolution for each symbol, take each lto::InputFile +/// and pass it and an array of symbol resolutions to the add() function. +/// - Call the getMaxTasks() function to get an upper bound on the number of +/// native object files that LTO may add to the link. +/// - Call the run() function. This function will use the supplied AddStream +/// function to add up to getMaxTasks() native object files to the link. +class LTO { + friend InputFile; + +public: + /// Create an LTO object. A default constructed LTO object has a reasonable + /// production configuration, but you can customize it by passing arguments to + /// this constructor. + /// FIXME: We do currently require the DiagHandler field to be set in Conf. + /// Until that is fixed, a Config argument is required. + LTO(Config Conf, ThinBackend Backend = nullptr, + unsigned ParallelCodeGenParallelismLevel = 1); + + /// Add an input file to the LTO link, using the provided symbol resolutions. + /// The symbol resolutions must appear in the enumeration order given by + /// InputFile::symbols(). + Error add(std::unique_ptr Obj, ArrayRef Res); + + /// Returns an upper bound on the number of tasks that the client may expect. + /// This may only be called after all IR object files have been added. For a + /// full description of tasks see include/llvm/CodeGen/LTOBackend.h. + size_t getMaxTasks() const; + + /// Runs the LTO pipeline. This function calls the supplied AddStream function + /// to add native object files to the link. + Error run(AddStreamFn AddStream); + +private: + std::string TheTriple; + const Target *TheTarget = nullptr; + + Config Conf; + + // These fields are for regular LTO and parallel LTO code generation. + unsigned ParallelCodeGenParallelismLevel; + LTOLLVMContext Ctx; + std::unique_ptr CombinedModule; + IRMover Mover; + + // These fields are for ThinLTO. + ThinBackend Backend; + ModuleSummaryIndex CombinedIndex; + StringMap ModuleMap; + + struct GlobalResolution { + /// The unmangled name of the global. + std::string IRName; + + bool UnnamedAddr = true; + + /// This field keeps track of the partition number of this global. The + /// regular LTO object is partition 0, while each ThinLTO object has its own + /// partition number from 1 onwards. + /// + /// Any global that is defined or used by more than one partition, or that + /// is referenced externally, may not be internalized. + /// + /// Partitions generally have a one-to-one correspondence with tasks, except + /// that we use partition 0 for all parallel LTO code generation partitions. + /// Any partitioning of the combined LTO object is done internally by the + /// LTO backend. + size_t Partition = Unknown; + + /// Special partition numbers. + enum { + /// A partition number has not yet been assigned to this global. + Unknown = -1ull, + + /// This global is either used by more than one partition or has an + /// external reference, and therefore cannot be internalized. + External = -2ull, + }; + }; + // Global mapping from mangled symbol names to resolutions. This is in + // particular necessary to track whether each symbol can be internalized. + // Because any input file may introduce a new cross-partition reference, we + // cannot make any final internalization decisions until all input files have + // been added and the client has called run(). During run() we apply + // internalization decisions either directly to the module (for regular LTO) + // or to the combined index (for ThinLTO; FIXME: not implemented yet). + StringMap GlobalResolutions; + + GlobalResolution &addSymbolToGlobalRes(object::IRObjectFile *Obj, + SmallPtrSet &Used, + const InputFile::Symbol &Sym, + SymbolResolution Res, + size_t Partition); + + Error addRegularLto(std::unique_ptr Input, + SmallPtrSet &Used, + ArrayRef Res); + Error addThinLto(std::unique_ptr Input, + SmallPtrSet &Used, + ArrayRef Res); + + Error runRegularLto(AddStreamFn AddStream); + Error runThinLto(AddStreamFn AddStream); + + void runThinLtoBackendThread(AddStreamFn AddStream, size_t Task, + MemoryBufferRef MBRef, + const FunctionImporter::ImportMapTy &ImportList); + + std::vector> ThinObjs; +}; + +struct SymbolResolution { + SymbolResolution() + : Prevailing(0), FinalDefinitionInLinkageUnit(0), VisibleToRegularObj(0) { + } + /// The linker has chosen this definition of the symbol. + unsigned Prevailing : 1; + + /// The definition of this symbol is unpreemptable at runtime and is known to + /// be in this linkage unit. + unsigned FinalDefinitionInLinkageUnit : 1; + + /// The definition of this symbol is visible outside of the LTO unit. + unsigned VisibleToRegularObj : 1; +}; + +} // namespace lto +} // namespace llvm + +#endif Index: lib/CodeGen/CMakeLists.txt =================================================================== --- lib/CodeGen/CMakeLists.txt +++ lib/CodeGen/CMakeLists.txt @@ -33,6 +33,7 @@ InterferenceCache.cpp InterleavedAccessPass.cpp IntrinsicLowering.cpp + LTOBackend.cpp LatencyPriorityQueue.cpp LexicalScopes.cpp LiveDebugValues.cpp Index: lib/CodeGen/LLVMBuild.txt =================================================================== --- lib/CodeGen/LLVMBuild.txt +++ lib/CodeGen/LLVMBuild.txt @@ -22,4 +22,4 @@ type = Library name = CodeGen parent = Libraries -required_libraries = Analysis BitReader BitWriter Core Instrumentation MC ProfileData Scalar Support Target TransformUtils +required_libraries = Analysis BitReader BitWriter Core IPO Instrumentation MC ProfileData Scalar Support Target TransformUtils Index: lib/CodeGen/LTOBackend.cpp =================================================================== --- /dev/null +++ lib/CodeGen/LTOBackend.cpp @@ -0,0 +1,196 @@ +#include "llvm/CodeGen/LTOBackend.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/MC/SubtargetFeature.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" +#include "llvm/Transforms/Utils/SplitModule.h" + +using namespace llvm; +using namespace lto; + +void lto::upgradeLinkage(GlobalValue *GV) { + switch (GV->getLinkage()) { + default: + break; + case GlobalValue::LinkOnceAnyLinkage: + GV->setLinkage(GlobalValue::WeakAnyLinkage); + break; + case GlobalValue::LinkOnceODRLinkage: + GV->setLinkage(GlobalValue::WeakODRLinkage); + break; + } +} + +namespace { + +static std::unique_ptr +createTargetMachine(Config &C, StringRef TheTriple, const Target *TheTarget) { + SubtargetFeatures Features; + Features.getDefaultSubtargetFeatures(Triple(TheTriple)); + for (const std::string &A : C.MAttrs) + Features.AddFeature(A); + + return std::unique_ptr(TheTarget->createTargetMachine( + TheTriple, C.CPU, Features.getString(), C.Options, C.RelocModel, + C.CodeModel, C.CGOptLevel)); +} + +bool opt(Config &C, StringRef TheTriple, const Target *TheTarget, size_t Task, + Module &M, bool IsThinLto) { + std::unique_ptr TM = + createTargetMachine(C, TheTriple, TheTarget); + + M.setDataLayout(TM->createDataLayout()); + + legacy::PassManager passes; + passes.add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); + + PassManagerBuilder PMB; + PMB.LibraryInfo = new TargetLibraryInfoImpl(Triple(TM->getTargetTriple())); + PMB.Inliner = createFunctionInliningPass(); + // Unconditionally verify input since it is not verified before this + // point and has unknown origin. + PMB.VerifyInput = true; + PMB.VerifyOutput = !C.DisableVerify; + PMB.LoopVectorize = true; + PMB.SLPVectorize = true; + PMB.OptLevel = C.OptLevel; + if (IsThinLto) + PMB.populateThinLTOPassManager(passes); + else + PMB.populateLTOPassManager(passes); + passes.run(M); + + if (C.PostOptModuleHook && !C.PostOptModuleHook(Task, M)) + return false; + + return true; +} + +bool codegen(Config &C, StringRef TheTriple, const Target *TheTarget, + AddStreamFn AddStream, size_t Task, Module &M) { + if (C.PreCodeGenModuleHook && !C.PreCodeGenModuleHook(Task, M)) + return false; + + std::unique_ptr TM = + createTargetMachine(C, TheTriple, TheTarget); + std::unique_ptr OS = AddStream(Task); + legacy::PassManager CodeGenPasses; + if (TM->addPassesToEmitFile(CodeGenPasses, *OS, + TargetMachine::CGFT_ObjectFile)) + report_fatal_error("Failed to setup codegen"); + CodeGenPasses.run(M); + return true; +} + +bool splitCodeGen(Config &C, StringRef TheTriple, const Target *TheTarget, + AddStreamFn AddStream, + unsigned ParallelCodeGenParallelismLevel, + std::unique_ptr M) { + ThreadPool CodegenThreadPool(ParallelCodeGenParallelismLevel); + unsigned ThreadCount = 0; + + SplitModule( + std::move(M), ParallelCodeGenParallelismLevel, + [&](std::unique_ptr MPart) { + // We want to clone the module in a new context to multi-thread the + // codegen. We do it by serializing partition modules to bitcode + // (while still on the main thread, in order to avoid data races) and + // spinning up new threads which deserialize the partitions into + // separate contexts. + // FIXME: Provide a more direct way to do this in LLVM. + SmallString<0> BC; + raw_svector_ostream BCOS(BC); + WriteBitcodeToFile(MPart.get(), BCOS); + + // Enqueue the task + CodegenThreadPool.async( + [&](const SmallString<0> &BC, unsigned ThreadId) { + LTOLLVMContext Ctx(C); + ErrorOr> MOrErr = parseBitcodeFile( + MemoryBufferRef(StringRef(BC.data(), BC.size()), + ""), + Ctx); + if (!MOrErr) + report_fatal_error("Failed to read bitcode"); + std::unique_ptr MPartInCtx = std::move(MOrErr.get()); + + codegen(C, TheTriple, TheTarget, AddStream, ThreadId, + *MPartInCtx); + }, + // Pass BC using std::move to ensure that it get moved rather than + // copied into the thread's context. + std::move(BC), ThreadCount++); + }, + false); + return true; +} + +} + +bool lto::backend(Config &Conf, StringRef TheTriple, const Target *TheTarget, + AddStreamFn AddStream, + unsigned ParallelCodeGenParallelismLevel, + std::unique_ptr M) { + if (!opt(Conf, TheTriple, TheTarget, 0, *M, /*IsThinLto=*/false)) + return false; + + if (ParallelCodeGenParallelismLevel == 1) + return codegen(Conf, TheTriple, TheTarget, AddStream, 0, *M); + else + return splitCodeGen(Conf, TheTriple, TheTarget, AddStream, + ParallelCodeGenParallelismLevel, std::move(M)); +} + +bool lto::thinBackend(Config &C, StringRef TheTriple, const Target *TheTarget, + size_t Task, AddStreamFn AddStream, Module &M, + ModuleSummaryIndex &CombinedIndex, + const FunctionImporter::ImportMapTy &ImportList, + StringMap &ModuleMap) { + if (C.PreOptModuleHook && !C.PreOptModuleHook(Task, M)) + return false; + + // FIXME: Apply symbol resolutions here. + for (Function &F : M) + upgradeLinkage(&F); + for (GlobalVariable &GV : M.globals()) + upgradeLinkage(&GV); + for (GlobalAlias &GA : M.aliases()) + upgradeLinkage(&GA); + + renameModuleForThinLTO(M, CombinedIndex); + + if (C.PostPromoteModuleHook && !C.PostPromoteModuleHook(Task, M)) + return false; + + // FIXME: Internalize based on summary info. + + if (C.PostInternalizeModuleHook && !C.PostInternalizeModuleHook(Task, M)) + return false; + + auto ModuleLoader = [&](StringRef Identifier) { + return std::move(getLazyBitcodeModule(MemoryBuffer::getMemBuffer( + ModuleMap[Identifier], false), + M.getContext(), + /*ShouldLazyLoadMetadata=*/true) + .get()); + }; + + FunctionImporter Importer(CombinedIndex, ModuleLoader); + Importer.importFunctions(M, ImportList); + + if (C.PostImportModuleHook && !C.PostImportModuleHook(Task, M)) + return false; + + if (!opt(C, TheTriple, TheTarget, Task, M, /*IsThinLto=*/true)) + return false; + + return codegen(C, TheTriple, TheTarget, AddStream, Task, M); +} Index: lib/LTO/CMakeLists.txt =================================================================== --- lib/LTO/CMakeLists.txt +++ lib/LTO/CMakeLists.txt @@ -48,6 +48,7 @@ add_llvm_library(LLVMLTO + LTO.cpp LTOModule.cpp LTOCodeGenerator.cpp UpdateCompilerUsed.cpp Index: lib/LTO/LTO.cpp =================================================================== --- /dev/null +++ lib/LTO/LTO.cpp @@ -0,0 +1,399 @@ +#include "llvm/LTO/LTO.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/CodeGen/Analysis.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Linker/IRMover.h" +#include "llvm/Object/ModuleSummaryIndexObjectFile.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/Utils/SplitModule.h" + +using namespace llvm; +using namespace lto; +using namespace object; + +namespace { + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class lto_error_category : public std::error_category { + const char *name() const LLVM_NOEXCEPT override { + return "llvm.lto"; + } + std::string message(int IE) const override { + lto_error E = static_cast(IE); + switch (E) { + case lto_error::comdat_alias: + return "Unable to determine comdat of alias!"; + case lto_error::unable_to_find_target: + return "Unable to find target"; + } + llvm_unreachable("invalid lto_error"); + } +}; + +static ManagedStatic error_category; + +} // end anonymous namespace + +const std::error_category <o::lto_category() { + return *error_category; +} + +ErrorOr> InputFile::create(MemoryBufferRef Object, + LTO &Lto) { + ErrorOr> Obj = + IRObjectFile::create(Object, Lto.Ctx); + if (!Obj) + return Obj.getError(); + return std::unique_ptr(new InputFile(std::move(*Obj))); +} + +LTO::LTO(Config Conf, ThinBackend Backend, + unsigned ParallelCodeGenParallelismLevel) + : Conf(Conf), Backend(Backend), + ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel), + Ctx(Conf), CombinedModule(make_unique("ld-temp.o", Ctx)), + Mover(*CombinedModule) { + if (!Backend) + this->Backend = createInProcessThinBackend(thread::hardware_concurrency()); +} + +// Add the given symbol to the GlobalResolutions map, and resolve its partition. +LTO::GlobalResolution <O::addSymbolToGlobalRes( + IRObjectFile *Obj, SmallPtrSet &Used, + const InputFile::Symbol &Sym, SymbolResolution Res, size_t Partition) { + GlobalValue *GV = Obj->getSymbolGV(Sym.I->getRawDataRefImpl()); + + auto &GlobalRes = GlobalResolutions[Sym.getName()]; + if (GV) + GlobalRes.UnnamedAddr &= GV->hasGlobalUnnamedAddr(); + if (Res.VisibleToRegularObj || (GV && Used.count(GV)) || + (GlobalRes.Partition != GlobalResolution::Unknown && + GlobalRes.Partition != Partition)) + GlobalRes.Partition = GlobalResolution::External; + else + GlobalRes.Partition = Partition; + return GlobalRes; +} + +Error LTO::add(std::unique_ptr Input, + ArrayRef Res) { + Module &M = Input->Obj->getModule(); + SmallPtrSet Used; + collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false); + + if (!Conf.OverrideTriple.empty()) + M.setTargetTriple(Conf.OverrideTriple); + else if (M.getTargetTriple().empty()) + M.setTargetTriple(Conf.DefaultTriple); + + if (TheTriple.empty()) { + TheTriple = M.getTargetTriple(); + std::string Msg; + TheTarget = TargetRegistry::lookupTarget(TheTriple, Msg); + if (!TheTarget) + return errorCodeToError(std::error_code( + static_cast(lto_error::unable_to_find_target), lto_category())); + } + + MemoryBufferRef MBRef = Input->Obj->getMemoryBufferRef(); + bool HasThinLtoSummary = hasGlobalValueSummary(MBRef, Conf.DiagHandler); + + if (HasThinLtoSummary) + return addThinLto(std::move(Input), Used, Res); + else + return addRegularLto(std::move(Input), Used, Res); +} + +// Add a regular LTO object to the link. +Error LTO::addRegularLto(std::unique_ptr Input, + SmallPtrSet &Used, + ArrayRef Res) { + Module &M = Input->Obj->getModule(); + M.materializeMetadata(); + UpgradeDebugInfo(M); + + std::vector Keep; + + for (GlobalVariable &GV : M.globals()) + if (GV.hasAppendingLinkage()) + Keep.push_back(&GV); + + auto ResI = Res.begin(); + for (const InputFile::Symbol &Sym : Input->symbols()) { + assert(ResI != Res.end()); + SymbolResolution Res = *ResI++; + GlobalResolution &GlobalRes = + addSymbolToGlobalRes(Input->Obj.get(), Used, Sym, Res, 0); + + GlobalValue *GV = Input->Obj->getSymbolGV(Sym.I->getRawDataRefImpl()); + if (Res.Prevailing && GV) { + GlobalRes.IRName = GV->getName(); + Keep.push_back(GV); + upgradeLinkage(GV); + } + + // FIXME: use proposed local attribute for FinalDefinitionInLinkageUnit. + } + assert(ResI == Res.end()); + + return Mover.move(Input->Obj->takeModule(), Keep, + [](GlobalValue &, IRMover::ValueAdder) {}); +} + +// Add a ThinLTO object to the link. +Error LTO::addThinLto(std::unique_ptr Input, + SmallPtrSet &Used, + ArrayRef Res) { + MemoryBufferRef MBRef = Input->Obj->getMemoryBufferRef(); + ErrorOr> + SummaryObjOrErr = + object::ModuleSummaryIndexObjectFile::create(MBRef, Conf.DiagHandler); + if (!SummaryObjOrErr) + return errorCodeToError(SummaryObjOrErr.getError()); + CombinedIndex.mergeFrom((*SummaryObjOrErr)->takeIndex(), ThinObjs.size()); + + auto ResI = Res.begin(); + for (const InputFile::Symbol &Sym : Input->symbols()) { + assert(ResI != Res.end()); + SymbolResolution Res = *ResI++; + addSymbolToGlobalRes(Input->Obj.get(), Used, Sym, Res, ThinObjs.size() + 1); + } + assert(ResI == Res.end()); + + ModuleMap[MBRef.getBufferIdentifier()] = MBRef; + ThinObjs.push_back(std::move(Input->Obj)); + return Error(); +} + +size_t LTO::getMaxTasks() const { + return ParallelCodeGenParallelismLevel + ThinObjs.size(); +} + +Error LTO::run(AddStreamFn AddStream) { + if (auto E = runRegularLto(AddStream)) + return std::move(E); + return runThinLto(AddStream); +} + +Error LTO::runRegularLto(AddStreamFn AddStream) { + if (Conf.PreOptModuleHook && !Conf.PreOptModuleHook(0, *CombinedModule)) + return Error(); + + for (const auto &R : GlobalResolutions) { + if (R.second.IRName.empty()) + continue; + GlobalValue *GV = CombinedModule->getNamedValue(R.second.IRName); + assert(GV); + assert(!GV->hasLocalLinkage() && + "Trying to internalize a symbol with local linkage!"); + GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global + : GlobalValue::UnnamedAddr::None); + if (R.second.Partition == 0) + GV->setLinkage(GlobalValue::InternalLinkage); + } + + if (Conf.PostInternalizeModuleHook && + !Conf.PostInternalizeModuleHook(0, *CombinedModule)) + return Error(); + + backend(Conf, TheTriple, TheTarget, AddStream, + ParallelCodeGenParallelismLevel, std::move(CombinedModule)); + return Error(); +} + +/// This class defines the interface to the ThinLTO backend. +class lto::ThinBackendProc { +protected: + Config &Conf; + ModuleSummaryIndex &CombinedIndex; + AddStreamFn AddStream; + +public: + ThinBackendProc(Config &Conf, ModuleSummaryIndex &CombinedIndex, + AddStreamFn AddStream) + : Conf(Conf), CombinedIndex(CombinedIndex), AddStream(AddStream) {} + + virtual ~ThinBackendProc() {} + virtual Error start(size_t Task, MemoryBufferRef MBRef, + StringMap &ImportLists, + StringMap &ModuleMap) = 0; + virtual void wait() = 0; +}; + +class InProcessThinBackend : public ThinBackendProc { + StringRef TheTriple; + const Target *TheTarget; + ThreadPool BackendThreadPool; + +public: + InProcessThinBackend(Config &Conf, StringRef TheTriple, + const Target *TheTarget, + ModuleSummaryIndex &CombinedIndex, + unsigned ThinLTOParallelismLevel, AddStreamFn AddStream) + : ThinBackendProc(Conf, CombinedIndex, AddStream), TheTriple(TheTriple), + TheTarget(TheTarget), BackendThreadPool(ThinLTOParallelismLevel) {} + + void runThinLtoBackendThread(AddStreamFn AddStream, size_t Task, + MemoryBufferRef MBRef, + ModuleSummaryIndex &CombinedIndex, + const FunctionImporter::ImportMapTy &ImportList, + StringMap &ModuleMap) { + LLVMContext BackendContext; + + ErrorOr> MOrErr = + parseBitcodeFile(MBRef, BackendContext); + assert(MOrErr && "Unable to load module in thread?"); + + thinBackend(Conf, TheTriple, TheTarget, Task, AddStream, **MOrErr, + CombinedIndex, ImportList, ModuleMap); + } + + Error start(size_t Task, MemoryBufferRef MBRef, + StringMap &ImportLists, + StringMap &ModuleMap) override { + StringRef ModulePath = MBRef.getBufferIdentifier(); + BackendThreadPool.async( + [=](MemoryBufferRef MBRef, + const FunctionImporter::ImportMapTy &ImportList, + ModuleSummaryIndex &CombinedIndex, + StringMap &ModuleMap) { + runThinLtoBackendThread(AddStream, Task, MBRef, CombinedIndex, + ImportList, ModuleMap); + }, + MBRef, std::ref(ImportLists[ModulePath]), std::ref(CombinedIndex), + std::ref(ModuleMap)); + return Error(); + } + + void wait() override { BackendThreadPool.wait(); } +}; + +ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) { + return [=](Config &Conf, StringRef TheTriple, const Target *TheTarget, + ModuleSummaryIndex &CombinedIndex, AddStreamFn AddStream) { + return make_unique( + Conf, TheTriple, TheTarget, CombinedIndex, ParallelismLevel, AddStream); + }; +} + +class WriteIndexesThinBackend : public ThinBackendProc { + std::string OldPrefix, NewPrefix; + bool ShouldEmitImportsFiles; + StringMap ModuleToDefinedGVSummaries; + +public: + WriteIndexesThinBackend(Config &Conf, ModuleSummaryIndex &CombinedIndex, + AddStreamFn AddStream, std::string OldPrefix, + std::string NewPrefix, bool ShouldEmitImportsFiles) + : ThinBackendProc(Conf, CombinedIndex, AddStream), OldPrefix(OldPrefix), + NewPrefix(NewPrefix), ShouldEmitImportsFiles(ShouldEmitImportsFiles) { + CombinedIndex.collectDefinedGVSummariesPerModule( + ModuleToDefinedGVSummaries); + } + + /// Given the original \p Path to an output file, replace any path + /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the + /// resulting directory if it does not yet exist. + std::string getThinLTOOutputFile(const std::string &Path, + const std::string &OldPrefix, + const std::string &NewPrefix) { + if (OldPrefix.empty() && NewPrefix.empty()) + return Path; + SmallString<128> NewPath(Path); + llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix); + StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str()); + if (!ParentPath.empty()) { + // Make sure the new directory exists, creating it if necessary. + if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath)) + llvm::errs() << "warning: could not create directory '" << ParentPath + << "': " << EC.message() << '\n'; + } + return NewPath.str(); + } + + Error start(size_t Task, MemoryBufferRef MBRef, + StringMap &ImportLists, + StringMap &ModuleMap) override { + StringRef ModulePath = MBRef.getBufferIdentifier(); + std::string NewModulePath = + getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix); + + std::map ModuleToSummariesForIndex; + gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, + ImportLists, ModuleToSummariesForIndex); + + std::error_code EC; + raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, + sys::fs::OpenFlags::F_None); + if (EC) + return errorCodeToError(EC); + WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); + + if (ShouldEmitImportsFiles) + return errorCodeToError(EmitImportsFiles( + ModulePath, NewModulePath + ".imports", ImportLists)); + return Error(); + } + + void wait() override {} +}; + +ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix, + std::string NewPrefix, + bool ShouldEmitImportsFiles) { + return [=](Config &Conf, StringRef TheTriple, const Target *TheTarget, + ModuleSummaryIndex &CombinedIndex, AddStreamFn AddStream) { + return make_unique(Conf, CombinedIndex, AddStream, + OldPrefix, NewPrefix, + ShouldEmitImportsFiles); + }; +} + +Error LTO::runThinLto(AddStreamFn AddStream) { + if (ThinObjs.empty()) + return Error(); + + if (Conf.CombinedIndexHook && !Conf.CombinedIndexHook(CombinedIndex)) + return Error(); + + // Collect for each module the list of function it defines (GUID -> + // Summary). + StringMap> + ModuleToDefinedGVSummaries(ThinObjs.size()); + CombinedIndex.collectDefinedGVSummariesPerModule( + ModuleToDefinedGVSummaries); + + StringMap ImportLists(ThinObjs.size()); + StringMap ExportLists(ThinObjs.size()); + ComputeCrossModuleImport(CombinedIndex, ModuleToDefinedGVSummaries, + ImportLists, ExportLists); + + std::unique_ptr BackendProc = + Backend(Conf, TheTriple, TheTarget, CombinedIndex, AddStream); + + size_t Task = ParallelCodeGenParallelismLevel; + size_t Partition = 1; + for (auto &ThinObj : ThinObjs) { + if (Error E = BackendProc->start(Task, ThinObj->getMemoryBufferRef(), + ImportLists, ModuleMap)) + return E; + + ++Task; + ++Partition; + } + + BackendProc->wait(); + return Error(); +} Index: lib/Object/IRObjectFile.cpp =================================================================== --- lib/Object/IRObjectFile.cpp +++ lib/Object/IRObjectFile.cpp @@ -323,5 +323,5 @@ return EC; std::unique_ptr &M = MOrErr.get(); - return llvm::make_unique(Object, std::move(M)); + return llvm::make_unique(BCOrErr.get(), std::move(M)); } Index: test/tools/gold/X86/coff.ll =================================================================== --- test/tools/gold/X86/coff.ll +++ test/tools/gold/X86/coff.ll @@ -16,7 +16,7 @@ ret void } -; CHECK: define internal void @h() local_unnamed_addr { +; CHECK: define internal void @h() { define linkonce_odr void @h() local_unnamed_addr { ret void } Index: test/tools/gold/X86/comdat.ll =================================================================== --- test/tools/gold/X86/comdat.ll +++ test/tools/gold/X86/comdat.ll @@ -3,6 +3,7 @@ ; RUN: %gold -shared -o %t3.o -plugin %llvmshlibdir/LLVMgold.so %t.o %t2.o \ ; RUN: -plugin-opt=save-temps ; RUN: llvm-dis %t3.o.bc -o - | FileCheck %s +; RUN: llvm-readobj -t %t3.o | FileCheck --check-prefix=OBJ %s $c1 = comdat any @@ -49,7 +50,16 @@ ; CHECK-DAG: @a23 = alias i32 (i8*), i32 (i8*)* @f1.2{{$}} ; CHECK-DAG: @a24 = alias i16, bitcast (i32 (i8*)* @f1.2 to i16*) -; CHECK: define weak_odr protected i32 @f1(i8*) comdat($c1) { +; OBJ: Name: f1 ( +; OBJ-NEXT: Value: +; OBJ-NEXT: Size: +; OBJ-NEXT: Binding: +; OBJ-NEXT: Type: +; OBJ-NEXT: Other [ +; OBJ-NEXT: STV_PROTECTED +; OBJ-NEXT: ] + +; CHECK: define weak_odr i32 @f1(i8*) comdat($c1) { ; CHECK-NEXT: bb10: ; CHECK-NEXT: br label %bb11{{$}} ; CHECK: bb11: Index: test/tools/gold/X86/common.ll =================================================================== --- test/tools/gold/X86/common.ll +++ test/tools/gold/X86/common.ll @@ -11,7 +11,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck %s --check-prefix=A ; Shared library case, we merge @a as common and keep it for the symbol table. -; A: @a = common global i32 0, align 8 +; A: @a = common global [4 x i8] zeroinitializer, align 8 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=emit-llvm \ @@ -19,7 +19,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck %s --check-prefix=B ; (i16 align 8) + (i8 align 16) = i16 align 16 -; B: @a = common global i16 0, align 16 +; B: @a = common global [2 x i8] zeroinitializer, align 16 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=emit-llvm \ @@ -27,7 +27,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck %s --check-prefix=C ; (i16 align 8) + (i8 align 1) = i16 align 8. -; C: @a = common global i16 0, align 8 +; C: @a = common global [2 x i8] zeroinitializer, align 8 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ ; RUN: --plugin-opt=emit-llvm \ @@ -35,7 +35,7 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck --check-prefix=EXEC %s ; All IR case, we internalize a after merging. -; EXEC: @a = internal global i32 0, align 8 +; EXEC: @a = internal global [4 x i8] zeroinitializer, align 8 ; RUN: llc %p/Inputs/common.ll -o %t2native.o -filetype=obj ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ @@ -44,4 +44,4 @@ ; RUN: llvm-dis %t3.o -o - | FileCheck --check-prefix=MIXED %s ; Mixed ELF and IR. We keep ours as common so the linker will finish the merge. -; MIXED: @a = common global i16 0, align 8 +; MIXED: @a = common global [2 x i8] zeroinitializer, align 8 Index: test/tools/gold/X86/emit-llvm.ll =================================================================== --- test/tools/gold/X86/emit-llvm.ll +++ test/tools/gold/X86/emit-llvm.ll @@ -25,19 +25,19 @@ target triple = "x86_64-unknown-linux-gnu" -; CHECK-DAG: @g1 = linkonce_odr constant i32 32 +; CHECK-DAG: @g1 = weak_odr constant i32 32 @g1 = linkonce_odr constant i32 32 -; CHECK-DAG: @g2 = internal local_unnamed_addr constant i32 32 +; CHECK-DAG: @g2 = internal constant i32 32 @g2 = linkonce_odr local_unnamed_addr constant i32 32 ; CHECK-DAG: @g3 = internal unnamed_addr constant i32 32 @g3 = linkonce_odr unnamed_addr constant i32 32 -; CHECK-DAG: @g4 = linkonce_odr global i32 32 +; CHECK-DAG: @g4 = weak_odr global i32 32 @g4 = linkonce_odr global i32 32 -; CHECK-DAG: @g5 = linkonce_odr local_unnamed_addr global i32 32 +; CHECK-DAG: @g5 = weak_odr global i32 32 @g5 = linkonce_odr local_unnamed_addr global i32 32 ; CHECK-DAG: @g6 = internal unnamed_addr global i32 32 @@ -75,8 +75,8 @@ ret void } -; CHECK-DAG: define linkonce_odr void @f5() -; OPT-DAG: define linkonce_odr void @f5() +; CHECK-DAG: define weak_odr void @f5() +; OPT-DAG: define weak_odr void @f5() define linkonce_odr void @f5() { ret void } Index: test/tools/gold/X86/parallel.ll =================================================================== --- test/tools/gold/X86/parallel.ll +++ test/tools/gold/X86/parallel.ll @@ -1,4 +1,5 @@ ; RUN: llvm-as -o %t.bc %s +; RUN: rm -f %t.opt.bc0 %t.opt.bc1 %t.o0 %t.o1 ; RUN: env LD_PRELOAD=%llvmshlibdir/LLVMgold.so %gold -plugin %llvmshlibdir/LLVMgold.so -u foo -u bar -plugin-opt jobs=2 -plugin-opt save-temps -m elf_x86_64 -o %t %t.bc ; RUN: llvm-dis %t.opt.bc0 -o - | FileCheck --check-prefix=CHECK-BC0 %s ; RUN: llvm-dis %t.opt.bc1 -o - | FileCheck --check-prefix=CHECK-BC1 %s Index: test/tools/gold/X86/start-lib-common.ll =================================================================== --- test/tools/gold/X86/start-lib-common.ll +++ test/tools/gold/X86/start-lib-common.ll @@ -19,4 +19,4 @@ ; Check that the common symbol is not dropped completely, which was a regression ; in r262676. -; CHECK: @x = common global i32 0 +; CHECK: @x = common global [4 x i8] zeroinitializer Index: test/tools/gold/X86/thinlto.ll =================================================================== --- test/tools/gold/X86/thinlto.ll +++ test/tools/gold/X86/thinlto.ll @@ -48,8 +48,8 @@ ; RUN: --plugin-opt=jobs=2 \ ; RUN: --plugin-opt=obj-path=%t5.o \ ; RUN: -shared %t.o %t2.o -o %t4 -; RUN: llvm-nm %t5.o0 | FileCheck %s --check-prefix=NM2 ; RUN: llvm-nm %t5.o1 | FileCheck %s --check-prefix=NM2 +; RUN: llvm-nm %t5.o2 | FileCheck %s --check-prefix=NM2 ; NM: T f ; NM2: T {{f|g}} Index: test/tools/gold/X86/visibility.ll =================================================================== --- test/tools/gold/X86/visibility.ll +++ test/tools/gold/X86/visibility.ll @@ -16,7 +16,7 @@ ; CHECK-NEXT: STV_PROTECTED ; CHECK-NEXT: ] -; IR: define protected void @foo +; IR: define void @foo define weak protected void @foo() { ret void Index: tools/gold/CMakeLists.txt =================================================================== --- tools/gold/CMakeLists.txt +++ tools/gold/CMakeLists.txt @@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} + LTO Linker BitWriter IPO Index: tools/gold/gold-plugin.cpp =================================================================== --- tools/gold/gold-plugin.cpp +++ tools/gold/gold-plugin.cpp @@ -28,6 +28,7 @@ #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" +#include "llvm/LTO/LTO.h" #include "llvm/Linker/IRMover.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Object/IRObjectFile.h" @@ -59,6 +60,7 @@ #define LDPT_GET_SYMBOLS_V3 28 using namespace llvm; +using namespace lto; static ld_plugin_status discard_message(int level, const char *format, ...) { // Die loudly. Recent versions of Gold pass ld_plugin_message as the first @@ -101,41 +103,17 @@ }; struct ResolutionInfo { - uint64_t CommonSize = 0; - unsigned CommonAlign = 0; - bool IsLinkonceOdr = true; - GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::Global; - GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility; - bool CommonInternal = false; - bool UseCommon = false; + bool CanOmitFromDynSym = true; + bool DefaultVisibility = true; }; -/// Class to own information used by a task or during its cleanup for a -/// ThinLTO backend instantiation. -class ThinLTOTaskInfo { - /// The input file holding the module bitcode read by the ThinLTO task. - PluginInputFile InputFile; - - /// The output stream the task will codegen into. - std::unique_ptr OS; - - /// The file name corresponding to the output stream, used during cleanup. - std::string Filename; - - /// Flag indicating whether the output file is a temp file that must be - /// added to the cleanup list during cleanup. - bool TempOutFile; - -public: - ThinLTOTaskInfo(PluginInputFile InputFile, std::unique_ptr OS, - std::string Filename, bool TempOutFile) - : InputFile(std::move(InputFile)), OS(std::move(OS)), Filename(Filename), - TempOutFile(TempOutFile) {} - - /// Performs task related cleanup activities that must be done - /// single-threaded (i.e. call backs to gold). - void cleanup(); +struct CommonResolution { + bool Prevailing = false; + bool VisibleToRegularObj = false; + uint64_t Size = 0; + unsigned Align = 0; }; + } static ld_plugin_add_symbols add_symbols = nullptr; @@ -143,13 +121,15 @@ static ld_plugin_add_input_file add_input_file = nullptr; static ld_plugin_set_extra_library_path set_extra_library_path = nullptr; static ld_plugin_get_view get_view = nullptr; +static bool IsExecutable = false; static Optional RelocationModel; static std::string output_name = ""; static std::list Modules; static StringMap ResInfo; +static std::map Commons; static std::vector Cleanup; static llvm::TargetOptions TargetOpts; -static std::string DefaultTriple = sys::getDefaultTargetTriple(); +static size_t MaxTasks; namespace options { enum OutputType { @@ -257,6 +237,14 @@ } } +static void saveBCFile(StringRef Path, Module &M) { + std::error_code EC; + raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::F_None); + if (EC) + message(LDPL_FATAL, "Failed to write the output file."); + WriteBitcodeToFile(&M, OS, /* ShouldPreserveUseListOrder */ false); +} + static ld_plugin_status claim_file_hook(const ld_plugin_input_file *file, int *claimed); static ld_plugin_status all_symbols_read_hook(void); @@ -291,10 +279,15 @@ switch (tv->tv_u.tv_val) { case LDPO_REL: // .o case LDPO_DYN: // .so + IsExecutable = false; + RelocationModel = Reloc::PIC_; + break; case LDPO_PIE: // position independent executable + IsExecutable = true; RelocationModel = Reloc::PIC_; break; case LDPO_EXEC: // .exe + IsExecutable = true; RelocationModel = Reloc::Static; break; default: @@ -388,20 +381,6 @@ return LDPS_OK; } -static const GlobalObject *getBaseObject(const GlobalValue &GV) { - if (auto *GA = dyn_cast(&GV)) - return GA->getBaseObject(); - return cast(&GV); -} - -static bool shouldSkip(uint32_t Symflags) { - if (!(Symflags & object::BasicSymbolRef::SF_Global)) - return true; - if (Symflags & object::BasicSymbolRef::SF_FormatSpecific) - return true; - return false; -} - static void diagnosticHandler(const DiagnosticInfo &DI) { if (const auto *BDI = dyn_cast(&DI)) { std::error_code EC = BDI->getError(); @@ -431,21 +410,18 @@ message(Level, "LLVM gold plugin: %s", ErrStorage.c_str()); } -static void diagnosticHandlerForContext(const DiagnosticInfo &DI, - void *Context) { - diagnosticHandler(DI); +static void check(Error E) { + handleAllErrors(std::move(E), [](ErrorInfoBase &EIB) { + message(LDPL_FATAL, "LLVM gold plugin: %s", EIB.message().c_str()); + return Error::success(); + }); } -static GlobalValue::VisibilityTypes -getMinVisibility(GlobalValue::VisibilityTypes A, - GlobalValue::VisibilityTypes B) { - if (A == GlobalValue::HiddenVisibility) - return A; - if (B == GlobalValue::HiddenVisibility) - return B; - if (A == GlobalValue::ProtectedVisibility) - return A; - return B; +template static T check(Expected E) { + if (E) + return std::move(*E); + check(E.takeError()); + return T(); } /// Called by gold to see whether this file is one that our plugin can handle. @@ -453,7 +429,12 @@ /// possible. static ld_plugin_status claim_file_hook(const ld_plugin_input_file *file, int *claimed) { - LLVMContext Context; + // FIXME: Remove this once we no longer need an LLVMContext to create an + // IRObjectFile. + Config TempConfig; + TempConfig.DiagHandler = diagnosticHandler; + LTO TempLto(TempConfig); + MemoryBufferRef BufferRef; std::unique_ptr Buffer; if (get_view) { @@ -482,9 +463,8 @@ BufferRef = Buffer->getMemBufferRef(); } - Context.setDiagnosticHandler(diagnosticHandlerForContext); - ErrorOr> ObjOrErr = - object::IRObjectFile::create(BufferRef, Context); + ErrorOr> ObjOrErr = + InputFile::create(BufferRef, TempLto); std::error_code EC = ObjOrErr.getError(); if (EC == object::object_error::invalid_file_type || EC == object::object_error::bitcode_section_not_found) @@ -497,82 +477,57 @@ EC.message().c_str()); return LDPS_ERR; } - std::unique_ptr Obj = std::move(*ObjOrErr); + std::unique_ptr Obj = std::move(*ObjOrErr); Modules.resize(Modules.size() + 1); claimed_file &cf = Modules.back(); cf.handle = file->handle; - // If we are doing ThinLTO compilation, don't need to process the symbols. - // Later we simply build a combined index file after all files are claimed. - if (options::thinlto && options::thinlto_index_only) - return LDPS_OK; - for (auto &Sym : Obj->symbols()) { uint32_t Symflags = Sym.getFlags(); - if (shouldSkip(Symflags)) - continue; cf.syms.push_back(ld_plugin_symbol()); ld_plugin_symbol &sym = cf.syms.back(); sym.version = nullptr; + StringRef Name = Sym.getName(); + sym.name = strdup(Name.str().c_str()); - SmallString<64> Name; - { - raw_svector_ostream OS(Name); - Sym.printName(OS); - } - sym.name = strdup(Name.c_str()); - - const GlobalValue *GV = Obj->getSymbolGV(Sym.getRawDataRefImpl()); + ResolutionInfo &Res = ResInfo[Name]; - ResolutionInfo &Res = ResInfo[sym.name]; + Res.CanOmitFromDynSym &= Sym.canBeOmittedFromSymbolTable(); sym.visibility = LDPV_DEFAULT; - if (GV) { - Res.UnnamedAddr = - GlobalValue::getMinUnnamedAddr(Res.UnnamedAddr, GV->getUnnamedAddr()); - Res.IsLinkonceOdr &= GV->hasLinkOnceLinkage(); - Res.Visibility = getMinVisibility(Res.Visibility, GV->getVisibility()); - switch (GV->getVisibility()) { - case GlobalValue::DefaultVisibility: - break; - case GlobalValue::HiddenVisibility: - sym.visibility = LDPV_HIDDEN; - break; - case GlobalValue::ProtectedVisibility: - sym.visibility = LDPV_PROTECTED; - break; - } + GlobalValue::VisibilityTypes Vis = Sym.getVisibility(); + if (Vis != GlobalValue::DefaultVisibility) + Res.DefaultVisibility = false; + switch (Vis) { + case GlobalValue::DefaultVisibility: + break; + case GlobalValue::HiddenVisibility: + sym.visibility = LDPV_HIDDEN; + break; + case GlobalValue::ProtectedVisibility: + sym.visibility = LDPV_PROTECTED; + break; } if (Symflags & object::BasicSymbolRef::SF_Undefined) { sym.def = LDPK_UNDEF; - if (GV && GV->hasExternalWeakLinkage()) + if (Symflags & object::BasicSymbolRef::SF_Weak) sym.def = LDPK_WEAKUNDEF; - } else { + } else if (Symflags & object::BasicSymbolRef::SF_Common) + sym.def = LDPK_COMMON; + else if (Symflags & object::BasicSymbolRef::SF_Weak) + sym.def = LDPK_WEAKDEF; + else sym.def = LDPK_DEF; - if (GV) { - assert(!GV->hasExternalWeakLinkage() && - !GV->hasAvailableExternallyLinkage() && "Not a declaration!"); - if (GV->hasCommonLinkage()) - sym.def = LDPK_COMMON; - else if (GV->isWeakForLinker()) - sym.def = LDPK_WEAKDEF; - } - } sym.size = 0; sym.comdat_key = nullptr; - if (GV) { - const GlobalObject *Base = getBaseObject(*GV); - if (!Base) - message(LDPL_FATAL, "Unable to determine comdat of alias!"); - const Comdat *C = Base->getComdat(); - if (C) - sym.comdat_key = strdup(C->getName().str().c_str()); - } + const Comdat *C = check(Sym.getComdat()); + if (C) + sym.comdat_key = strdup(C->getName().str().c_str()); sym.resolution = LDPR_UNKNOWN; } @@ -587,13 +542,6 @@ return LDPS_OK; } -static void internalize(GlobalValue &GV) { - if (GV.isDeclarationForLinker()) - return; // We get here if there is a matching asm definition. - if (!GV.hasLocalLinkage()) - GV.setLinkage(GlobalValue::InternalLinkage); -} - static const char *getResolutionName(ld_plugin_symbol_resolution R) { switch (R) { case LDPR_UNKNOWN: @@ -643,68 +591,24 @@ return View; } -static std::unique_ptr -getModuleSummaryIndexForFile(claimed_file &F, ld_plugin_input_file &Info) { - const void *View = getSymbolsAndView(F); - if (!View) - return nullptr; - - MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize), - Info.name); - - // Don't bother trying to build an index if there is no summary information - // in this bitcode file. - if (!object::ModuleSummaryIndexObjectFile::hasGlobalValueSummaryInMemBuffer( - BufferRef, diagnosticHandler)) - return std::unique_ptr(nullptr); - - ErrorOr> ObjOrErr = - object::ModuleSummaryIndexObjectFile::create(BufferRef, - diagnosticHandler); - - if (std::error_code EC = ObjOrErr.getError()) - message(LDPL_FATAL, - "Could not read module summary index bitcode from file : %s", - EC.message().c_str()); - - object::ModuleSummaryIndexObjectFile &Obj = **ObjOrErr; - - return Obj.takeIndex(); -} - -static std::unique_ptr -getModuleForFile(LLVMContext &Context, claimed_file &F, const void *View, - ld_plugin_input_file &Info, raw_fd_ostream *ApiFile, - StringSet<> &Internalize, std::vector &Keep, - StringMap &Realign) { +static void addModule(LTO &Lto, claimed_file &F, const void *View, + ld_plugin_input_file &Info, raw_fd_ostream *ApiFile) { MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize), Info.name); - ErrorOr> ObjOrErr = - object::IRObjectFile::create(BufferRef, Context); + ErrorOr> ObjOrErr = + InputFile::create(BufferRef, Lto); if (std::error_code EC = ObjOrErr.getError()) message(LDPL_FATAL, "Could not read bitcode from file : %s", EC.message().c_str()); - object::IRObjectFile &Obj = **ObjOrErr; - - Module &M = Obj.getModule(); - - M.materializeMetadata(); - UpgradeDebugInfo(M); - - SmallPtrSet Used; - collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false); + InputFile &Obj = **ObjOrErr; unsigned SymNum = 0; + std::vector Resols(F.syms.size()); for (auto &ObjSym : Obj.symbols()) { - GlobalValue *GV = Obj.getSymbolGV(ObjSym.getRawDataRefImpl()); - if (GV && GV->hasAppendingLinkage()) - Keep.push_back(GV); - - if (shouldSkip(ObjSym.getFlags())) - continue; ld_plugin_symbol &Sym = F.syms[SymNum]; + SymbolResolution &R = Resols[SymNum]; ++SymNum; ld_plugin_symbol_resolution Resolution = @@ -713,62 +617,7 @@ if (options::generate_api_file) *ApiFile << Sym.name << ' ' << getResolutionName(Resolution) << '\n'; - if (!GV) { - freeSymName(Sym); - continue; // Asm symbol. - } - ResolutionInfo &Res = ResInfo[Sym.name]; - if (Resolution == LDPR_PREVAILING_DEF_IRONLY_EXP && !Res.IsLinkonceOdr) - Resolution = LDPR_PREVAILING_DEF; - - // In ThinLTO mode change all prevailing resolutions to LDPR_PREVAILING_DEF. - // For ThinLTO the IR files are compiled through the backend independently, - // so we need to ensure that any prevailing linkonce copy will be emitted - // into the object file by making it weak. Additionally, we can skip the - // IRONLY handling for internalization, which isn't performed in ThinLTO - // mode currently anyway. - if (options::thinlto && (Resolution == LDPR_PREVAILING_DEF_IRONLY_EXP || - Resolution == LDPR_PREVAILING_DEF_IRONLY)) - Resolution = LDPR_PREVAILING_DEF; - - GV->setUnnamedAddr(Res.UnnamedAddr); - GV->setVisibility(Res.Visibility); - - // Override gold's resolution for common symbols. We want the largest - // one to win. - if (GV->hasCommonLinkage()) { - if (Resolution == LDPR_PREVAILING_DEF_IRONLY) - Res.CommonInternal = true; - - if (Resolution == LDPR_PREVAILING_DEF_IRONLY || - Resolution == LDPR_PREVAILING_DEF) - Res.UseCommon = true; - - const DataLayout &DL = GV->getParent()->getDataLayout(); - uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType()); - unsigned Align = GV->getAlignment(); - - if (Res.UseCommon && Size >= Res.CommonSize) { - // Take GV. - if (Res.CommonInternal) - Resolution = LDPR_PREVAILING_DEF_IRONLY; - else - Resolution = LDPR_PREVAILING_DEF; - cast(GV)->setAlignment( - std::max(Res.CommonAlign, Align)); - } else { - // Do not take GV, it's smaller than what we already have in the - // combined module. - Resolution = LDPR_PREEMPTED_IR; - if (Align > Res.CommonAlign) - // Need to raise the alignment though. - Realign[Sym.name] = Align; - } - - Res.CommonSize = std::max(Res.CommonSize, Size); - Res.CommonAlign = std::max(Res.CommonAlign, Align); - } switch (Resolution) { case LDPR_UNKNOWN: @@ -779,57 +628,47 @@ case LDPR_RESOLVED_DYN: case LDPR_PREEMPTED_IR: case LDPR_PREEMPTED_REG: - break; - case LDPR_UNDEF: - if (!GV->isDeclarationForLinker()) - assert(GV->hasComdat()); break; - case LDPR_PREVAILING_DEF_IRONLY: { - Keep.push_back(GV); - // The IR linker has to be able to map this value to a declaration, - // so we can only internalize after linking. - if (!Used.count(GV)) - Internalize.insert(GV->getName()); + case LDPR_PREVAILING_DEF_IRONLY: + R.Prevailing = true; break; - } case LDPR_PREVAILING_DEF: - Keep.push_back(GV); - // There is a non IR use, so we have to force optimizations to keep this. - switch (GV->getLinkage()) { - default: - break; - case GlobalValue::LinkOnceAnyLinkage: - GV->setLinkage(GlobalValue::WeakAnyLinkage); - break; - case GlobalValue::LinkOnceODRLinkage: - GV->setLinkage(GlobalValue::WeakODRLinkage); - break; - } + R.Prevailing = true; + R.VisibleToRegularObj = true; break; - case LDPR_PREVAILING_DEF_IRONLY_EXP: { - Keep.push_back(GV); - if (canBeOmittedFromSymbolTable(GV)) - Internalize.insert(GV->getName()); + case LDPR_PREVAILING_DEF_IRONLY_EXP: + R.Prevailing = true; + if (!Res.CanOmitFromDynSym) + R.VisibleToRegularObj = true; break; } + + if (Resolution != LDPR_RESOLVED_DYN && Resolution != LDPR_UNDEF && + (IsExecutable || !Res.DefaultVisibility)) + R.FinalDefinitionInLinkageUnit = true; + + if (ObjSym.getFlags() & object::BasicSymbolRef::SF_Common) { + // We ignore gold's resolution for common symbols. A common symbol with + // the correct size and alignment is added to the module by the pre-opt + // module hook if any common symbol prevailed. + CommonResolution &CommonRes = Commons[ObjSym.getIRName()]; + if (R.Prevailing) { + CommonRes.Prevailing = true; + CommonRes.VisibleToRegularObj = R.VisibleToRegularObj; + } + CommonRes.Size = std::max(CommonRes.Size, ObjSym.getCommonSize()); + CommonRes.Align = std::max(CommonRes.Align, ObjSym.getCommonAlignment()); + R.Prevailing = false; } freeSymName(Sym); } - return Obj.takeModule(); -} - -static void saveBCFile(StringRef Path, Module &M) { - std::error_code EC; - raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::F_None); - if (EC) - message(LDPL_FATAL, "Failed to write the output file."); - WriteBitcodeToFile(&M, OS, /* ShouldPreserveUseListOrder */ false); + check(Lto.add(std::move(*ObjOrErr), Resols)); } static void recordFile(std::string Filename, bool TempOutFile) { @@ -841,151 +680,6 @@ Cleanup.push_back(Filename.c_str()); } -void ThinLTOTaskInfo::cleanup() { - // Close the output file descriptor before we pass it to gold. - OS->close(); - - recordFile(Filename, TempOutFile); -} - -namespace { -/// Class to manage optimization and code generation for a module, possibly -/// in a thread (ThinLTO). -class CodeGen { - /// The module for which this will generate code. - std::unique_ptr M; - - /// The output stream to generate code into. - raw_fd_ostream *OS; - - /// The task ID when this was invoked in a thread (ThinLTO). - int TaskID; - - /// The module summary index for ThinLTO tasks. - const ModuleSummaryIndex *CombinedIndex; - - /// The target machine for generating code for this module. - std::unique_ptr TM; - - /// Filename to use as base when save-temps is enabled, used to get - /// a unique and identifiable save-temps output file for each ThinLTO backend. - std::string SaveTempsFilename; - -public: - /// Constructor used by full LTO. - CodeGen(std::unique_ptr M) - : M(std::move(M)), OS(nullptr), TaskID(-1), CombinedIndex(nullptr) { - initTargetMachine(); - } - /// Constructor used by ThinLTO. - CodeGen(std::unique_ptr M, raw_fd_ostream *OS, int TaskID, - const ModuleSummaryIndex *CombinedIndex, std::string Filename) - : M(std::move(M)), OS(OS), TaskID(TaskID), CombinedIndex(CombinedIndex), - SaveTempsFilename(Filename) { - assert(options::thinlto == !!CombinedIndex && - "Expected module summary index iff performing ThinLTO"); - initTargetMachine(); - } - - /// Invoke LTO passes and the code generator for the module. - void runAll(); - - /// Invoke the actual code generation to emit Module's object to file. - void runCodegenPasses(); - -private: - const Target *TheTarget; - std::string TripleStr; - std::string FeaturesString; - TargetOptions Options; - - /// Create a target machine for the module. Must be unique for each - /// module/task. - void initTargetMachine(); - - std::unique_ptr createTargetMachine(); - - /// Run all LTO passes on the module. - void runLTOPasses(); - - /// Sets up output files necessary to perform optional multi-threaded - /// split code generation, and invokes the code generation implementation. - /// If BCFileName is not empty, saves bitcode for module partitions into - /// {BCFileName}0 .. {BCFileName}N. - void runSplitCodeGen(const SmallString<128> &BCFilename); -}; -} - -static SubtargetFeatures getFeatures(Triple &TheTriple) { - SubtargetFeatures Features; - Features.getDefaultSubtargetFeatures(TheTriple); - for (const std::string &A : MAttrs) - Features.AddFeature(A); - return Features; -} - -static CodeGenOpt::Level getCGOptLevel() { - switch (options::OptLevel) { - case 0: - return CodeGenOpt::None; - case 1: - return CodeGenOpt::Less; - case 2: - return CodeGenOpt::Default; - case 3: - return CodeGenOpt::Aggressive; - } - llvm_unreachable("Invalid optimization level"); -} - -void CodeGen::initTargetMachine() { - TripleStr = M->getTargetTriple(); - Triple TheTriple(TripleStr); - - std::string ErrMsg; - TheTarget = TargetRegistry::lookupTarget(TripleStr, ErrMsg); - if (!TheTarget) - message(LDPL_FATAL, "Target not found: %s", ErrMsg.c_str()); - - SubtargetFeatures Features = getFeatures(TheTriple); - FeaturesString = Features.getString(); - Options = InitTargetOptionsFromCodeGenFlags(); - - TM = createTargetMachine(); -} - -std::unique_ptr CodeGen::createTargetMachine() { - CodeGenOpt::Level CGOptLevel = getCGOptLevel(); - - return std::unique_ptr(TheTarget->createTargetMachine( - TripleStr, options::mcpu, FeaturesString, Options, RelocationModel, - CodeModel::Default, CGOptLevel)); -} - -void CodeGen::runLTOPasses() { - M->setDataLayout(TM->createDataLayout()); - - legacy::PassManager passes; - passes.add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); - - PassManagerBuilder PMB; - PMB.LibraryInfo = new TargetLibraryInfoImpl(Triple(TM->getTargetTriple())); - PMB.Inliner = createFunctionInliningPass(); - // Unconditionally verify input since it is not verified before this - // point and has unknown origin. - PMB.VerifyInput = true; - PMB.VerifyOutput = !options::DisableVerify; - PMB.LoopVectorize = true; - PMB.SLPVectorize = true; - PMB.OptLevel = options::OptLevel; - PMB.ModuleSummary = CombinedIndex; - if (options::thinlto) - PMB.populateThinLTOPassManager(passes); - else - PMB.populateLTOPassManager(passes); - passes.run(*M); -} - /// Open a file and return the new file descriptor given a base input /// file name, a flag indicating whether a temp file should be generated, /// and an optional task id. The new filename generated is @@ -1011,203 +705,44 @@ return FD; } -void CodeGen::runCodegenPasses() { - assert(OS && "Output stream must be set before emitting to file"); - legacy::PassManager CodeGenPasses; - if (TM->addPassesToEmitFile(CodeGenPasses, *OS, - TargetMachine::CGFT_ObjectFile)) - report_fatal_error("Failed to setup codegen"); - CodeGenPasses.run(*M); -} - -void CodeGen::runSplitCodeGen(const SmallString<128> &BCFilename) { - SmallString<128> Filename; - // Note that openOutputFile will append a unique ID for each task - if (!options::obj_path.empty()) - Filename = options::obj_path; - else if (options::TheOutputType == options::OT_SAVE_TEMPS) - Filename = output_name + ".o"; - - // Note that the default parallelism is 1 instead of the - // hardware_concurrency, as there are behavioral differences between - // parallelism levels (e.g. symbol ordering will be different, and some uses - // of inline asm currently have issues with parallelism >1). - unsigned int MaxThreads = options::Parallelism ? options::Parallelism : 1; - - std::vector> Filenames(MaxThreads); - std::vector> BCFilenames(MaxThreads); - bool TempOutFile = Filename.empty(); - { - // Open a file descriptor for each backend task. This is done in a block - // so that the output file descriptors are closed before gold opens them. - std::list OSs; - std::vector OSPtrs(MaxThreads); - for (unsigned I = 0; I != MaxThreads; ++I) { - int FD = openOutputFile(Filename, TempOutFile, Filenames[I], - // Only append ID if there are multiple tasks. - MaxThreads > 1 ? I : -1); - OSs.emplace_back(FD, true); - OSPtrs[I] = &OSs.back(); - } - - std::list BCOSs; - std::vector BCOSPtrs; - if (!BCFilename.empty() && MaxThreads > 1) { - for (unsigned I = 0; I != MaxThreads; ++I) { - int FD = openOutputFile(BCFilename, false, BCFilenames[I], I); - BCOSs.emplace_back(FD, true); - BCOSPtrs.push_back(&BCOSs.back()); - } - } - - // Run backend tasks. - splitCodeGen(std::move(M), OSPtrs, BCOSPtrs, - [&]() { return createTargetMachine(); }); - } - - for (auto &Filename : Filenames) - recordFile(Filename.c_str(), TempOutFile); -} - -void CodeGen::runAll() { - runLTOPasses(); - - SmallString<128> OptFilename; - if (options::TheOutputType == options::OT_SAVE_TEMPS) { - OptFilename = output_name; - // If the CodeGen client provided a filename, use it. Always expect - // a provided filename if we are in a task (i.e. ThinLTO backend). - assert(!SaveTempsFilename.empty() || TaskID == -1); - if (!SaveTempsFilename.empty()) - OptFilename = SaveTempsFilename; - OptFilename += ".opt.bc"; - saveBCFile(OptFilename, *M); - } - - // If we are already in a thread (i.e. ThinLTO), just perform - // codegen passes directly. - if (TaskID >= 0) - runCodegenPasses(); - // Otherwise attempt split code gen. - else - runSplitCodeGen(OptFilename); -} - -/// Links the module in \p View from file \p F into the combined module -/// saved in the IRMover \p L. Returns true on error, false on success. -static bool linkInModule(LLVMContext &Context, IRMover &L, claimed_file &F, - const void *View, ld_plugin_input_file &File, - raw_fd_ostream *ApiFile, StringSet<> &Internalize) { - std::vector Keep; - StringMap Realign; - std::unique_ptr M = getModuleForFile( - Context, F, View, File, ApiFile, Internalize, Keep, Realign); - if (!M.get()) - return false; - if (!options::triple.empty()) - M->setTargetTriple(options::triple.c_str()); - else if (M->getTargetTriple().empty()) { - M->setTargetTriple(DefaultTriple); - } - - if (L.move(std::move(M), Keep, [](GlobalValue &, IRMover::ValueAdder) {})) - return true; - - for (const auto &I : Realign) { - GlobalValue *Dst = L.getModule().getNamedValue(I.first()); - if (!Dst) +/// Add all required common symbols to M, which is expected to be the first +/// combined module. +static void addCommons(Module &M) { + for (auto &I : Commons) { + if (!I.second.Prevailing) continue; - cast(Dst)->setAlignment(I.second); + ArrayType *Ty = + ArrayType::get(Type::getInt8Ty(M.getContext()), I.second.Size); + GlobalVariable *OldGV = M.getNamedGlobal(I.first); + auto *GV = new GlobalVariable(M, Ty, false, GlobalValue::CommonLinkage, + ConstantAggregateZero::get(Ty), ""); + GV->setAlignment(I.second.Align); + if (OldGV) { + OldGV->replaceAllUsesWith(ConstantExpr::getBitCast(GV, OldGV->getType())); + GV->takeName(OldGV); + OldGV->eraseFromParent(); + } else { + GV->setName(I.first); + } + // We may only internalize commons if there is a single LTO task because + // other native object files may require the common. + if (MaxTasks == 1 && !I.second.VisibleToRegularObj) + GV->setLinkage(GlobalValue::InternalLinkage); } - - return false; -} - -/// Perform the ThinLTO backend on a single module, invoking the LTO and codegen -/// pipelines. -static void thinLTOBackendTask(claimed_file &F, const void *View, - ld_plugin_input_file &File, - raw_fd_ostream *ApiFile, - const ModuleSummaryIndex &CombinedIndex, - raw_fd_ostream *OS, unsigned TaskID) { - // Need to use a separate context for each task - LLVMContext Context; - Context.setDiscardValueNames(options::TheOutputType != - options::OT_SAVE_TEMPS); - Context.enableDebugTypeODRUniquing(); // Merge debug info types. - Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true); - - std::unique_ptr NewModule(new llvm::Module(File.name, Context)); - IRMover L(*NewModule.get()); - - StringSet<> Dummy; - if (linkInModule(Context, L, F, View, File, ApiFile, Dummy)) - message(LDPL_FATAL, "Failed to rename module for ThinLTO"); - if (renameModuleForThinLTO(*NewModule, CombinedIndex)) - message(LDPL_FATAL, "Failed to rename module for ThinLTO"); - - CodeGen codeGen(std::move(NewModule), OS, TaskID, &CombinedIndex, File.name); - codeGen.runAll(); } -/// Launch each module's backend pipeline in a separate task in a thread pool. -static void thinLTOBackends(raw_fd_ostream *ApiFile, - const ModuleSummaryIndex &CombinedIndex) { - unsigned TaskCount = 0; - std::vector Tasks; - Tasks.reserve(Modules.size()); - unsigned int MaxThreads = options::Parallelism - ? options::Parallelism - : thread::hardware_concurrency(); - - // Create ThreadPool in nested scope so that threads will be joined - // on destruction. - { - ThreadPool ThinLTOThreadPool(MaxThreads); - for (claimed_file &F : Modules) { - // Do all the gold callbacks in the main thread, since gold is not thread - // safe by default. - PluginInputFile InputFile(F.handle); - const void *View = getSymbolsAndView(F); - if (!View) - continue; - - SmallString<128> Filename; - if (!options::obj_path.empty()) - // Note that openOutputFile will append a unique ID for each task - Filename = options::obj_path; - else if (options::TheOutputType == options::OT_SAVE_TEMPS) { - // Use the input file name so that we get a unique and identifiable - // output file for each ThinLTO backend task. - Filename = InputFile.file().name; - Filename += ".thinlto.o"; - } - bool TempOutFile = Filename.empty(); - - SmallString<128> NewFilename; - int FD = openOutputFile(Filename, TempOutFile, NewFilename, - // Only append the TaskID if we will use the - // non-unique obj_path. - !options::obj_path.empty() ? TaskCount : -1); - TaskCount++; - std::unique_ptr OS = - llvm::make_unique(FD, true); - - // Enqueue the task - ThinLTOThreadPool.async(thinLTOBackendTask, std::ref(F), View, - std::ref(InputFile.file()), ApiFile, - std::ref(CombinedIndex), OS.get(), TaskCount); - - // Record the information needed by the task or during its cleanup - // to a ThinLTOTaskInfo instance. For information needed by the task - // the unique_ptr ownership is transferred to the ThinLTOTaskInfo. - Tasks.emplace_back(std::move(InputFile), std::move(OS), - NewFilename.c_str(), TempOutFile); - } +static CodeGenOpt::Level getCGOptLevel() { + switch (options::OptLevel) { + case 0: + return CodeGenOpt::None; + case 1: + return CodeGenOpt::Less; + case 2: + return CodeGenOpt::Default; + case 3: + return CodeGenOpt::Aggressive; } - - for (auto &Task : Tasks) - Task.cleanup(); + llvm_unreachable("Invalid optimization level"); } /// Parse the thinlto_prefix_replace option into the \p OldPrefix and @@ -1221,118 +756,74 @@ NewPrefix = Split.second.str(); } -/// Given the original \p Path to an output file, replace any path -/// prefix matching \p OldPrefix with \p NewPrefix. Also, create the -/// resulting directory if it does not yet exist. -static std::string getThinLTOOutputFile(const std::string &Path, - const std::string &OldPrefix, - const std::string &NewPrefix) { - if (OldPrefix.empty() && NewPrefix.empty()) - return Path; - SmallString<128> NewPath(Path); - llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix); - StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str()); - if (!ParentPath.empty()) { - // Make sure the new directory exists, creating it if necessary. - if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath)) - llvm::errs() << "warning: could not create directory '" << ParentPath - << "': " << EC.message() << '\n'; - } - return NewPath.str(); -} - -/// Perform ThinLTO link, which creates the combined index file. -/// Also, either launch backend threads or (under thinlto-index-only) -/// emit individual index files for distributed backends and exit. -static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) { - ModuleSummaryIndex CombinedIndex; - uint64_t NextModuleId = 0; - for (claimed_file &F : Modules) { - PluginInputFile InputFile(F.handle); - - std::unique_ptr Index = - getModuleSummaryIndexForFile(F, InputFile.file()); - - // Skip files without a module summary. - if (Index) - CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId); +static std::unique_ptr createLTO() { + Config Conf; + ThinBackend Backend; + unsigned ParallelCodeGenParallelismLevel = 1; + + Conf.CPU = options::mcpu; + Conf.Options = InitTargetOptionsFromCodeGenFlags(); + Conf.MAttrs = MAttrs; + Conf.RelocModel = *RelocationModel; + Conf.CGOptLevel = getCGOptLevel(); + Conf.DisableVerify = options::DisableVerify; + Conf.OptLevel = options::OptLevel; + if (options::Parallelism) { + if (options::thinlto) + Backend = createInProcessThinBackend(options::Parallelism); + else + ParallelCodeGenParallelismLevel = options::Parallelism; } - - if (options::thinlto_emit_imports_files && !options::thinlto_index_only) - message(LDPL_WARNING, - "thinlto-emit-imports-files ignored unless thinlto-index-only"); - if (options::thinlto_index_only) { - // Collect for each module the list of function it defines (GUID -> - // Summary). - StringMap> - ModuleToDefinedGVSummaries(NextModuleId); - CombinedIndex.collectDefinedGVSummariesPerModule( - ModuleToDefinedGVSummaries); - - // FIXME: We want to do this for the case where the threads are launched - // from gold as well, in which case this will be moved out of the - // thinlto_index_only handling, and the function importer will be invoked - // directly using the Lists. - StringMap ImportLists(NextModuleId); - StringMap ExportLists(NextModuleId); - ComputeCrossModuleImport(CombinedIndex, ModuleToDefinedGVSummaries, - ImportLists, ExportLists); - - // If the thinlto-prefix-replace option was specified, parse it and - // extract the old and new prefixes. std::string OldPrefix, NewPrefix; getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix); - - // For each input bitcode file, generate an individual index that - // contains summaries only for its own global values, and for any that - // should be imported. - for (claimed_file &F : Modules) { - PluginInputFile InputFile(F.handle); - std::error_code EC; - - std::string NewModulePath = - getThinLTOOutputFile(InputFile.file().name, OldPrefix, NewPrefix); - raw_fd_ostream OS((Twine(NewModulePath) + ".thinlto.bc").str(), EC, - sys::fs::OpenFlags::F_None); - if (EC) - message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", - NewModulePath.c_str(), EC.message().c_str()); - // Build a map of module to the GUIDs and summary objects that should - // be written to its index. - std::map ModuleToSummariesForIndex; - gatherImportedSummariesForModule(InputFile.file().name, - ModuleToDefinedGVSummaries, ImportLists, - ModuleToSummariesForIndex); - WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); - - if (options::thinlto_emit_imports_files) { - if ((EC = EmitImportsFiles( - InputFile.file().name, - (Twine(NewModulePath) + ".imports").str(), - ImportLists))) - message(LDPL_FATAL, "Unable to open %s.imports", - NewModulePath.c_str(), EC.message().c_str()); - } + Backend = createWriteIndexesThinBackend( + OldPrefix, NewPrefix, options::thinlto_emit_imports_files); + } + + Conf.OverrideTriple = options::triple; + Conf.DefaultTriple = sys::getDefaultTargetTriple(); + + Conf.DiagHandler = diagnosticHandler; + Conf.ShouldDiscardValueNames = + (options::TheOutputType != options::OT_SAVE_TEMPS); + + Conf.PostInternalizeModuleHook = [](size_t Task, Module &M) -> bool { + if (Task == 0) + addCommons(M); + + if (options::TheOutputType == options::OT_DISABLE) + return false; + + if (options::TheOutputType != options::OT_NORMAL) { + std::string path; + if (options::TheOutputType == options::OT_BC_ONLY) + path = output_name; + else + path = output_name + ".bc"; + if (MaxTasks != 1) + path += utostr(Task); + saveBCFile(path, M); + if (options::TheOutputType == options::OT_BC_ONLY) + return false; } - cleanup_hook(); - exit(0); - } + return true; + }; - // Create OS in nested scope so that it will be closed on destruction. - { - std::error_code EC; - raw_fd_ostream OS(output_name + ".thinlto.bc", EC, - sys::fs::OpenFlags::F_None); - if (EC) - message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", - output_name.data(), EC.message().c_str()); - WriteIndexToFile(CombinedIndex, OS); - } + Conf.PreCodeGenModuleHook = [](size_t Task, Module &M) -> bool { + SmallString<128> OptFilename; + if (options::TheOutputType == options::OT_SAVE_TEMPS) { + OptFilename = output_name + ".opt.bc"; + if (MaxTasks != 1) + OptFilename += utostr(Task); + saveBCFile(OptFilename, M); + } - thinLTOBackends(ApiFile, CombinedIndex); - return LDPS_OK; + return true; + }; + + return make_unique(Conf, Backend, ParallelCodeGenParallelismLevel); } /// gold informs us that all symbols have been read. At this point, we use @@ -1345,51 +836,50 @@ if (unsigned NumOpts = options::extra.size()) cl::ParseCommandLineOptions(NumOpts, &options::extra[0]); - if (options::thinlto) - return thinLTOLink(ApiFile); - - LLVMContext Context; - Context.setDiscardValueNames(options::TheOutputType != - options::OT_SAVE_TEMPS); - Context.enableDebugTypeODRUniquing(); // Merge debug info types. - Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true); + std::unique_ptr Lto = createLTO(); - std::unique_ptr Combined(new Module("ld-temp.o", Context)); - IRMover L(*Combined); - - StringSet<> Internalize; for (claimed_file &F : Modules) { PluginInputFile InputFile(F.handle); const void *View = getSymbolsAndView(F); if (!View) continue; - if (linkInModule(Context, L, F, View, InputFile.file(), ApiFile, - Internalize)) - message(LDPL_FATAL, "Failed to link module"); + addModule(*Lto, F, View, InputFile.file(), ApiFile); } - for (const auto &Name : Internalize) { - GlobalValue *GV = Combined->getNamedValue(Name.first()); - if (GV) - internalize(*GV); - } + SmallString<128> Filename; + // Note that openOutputFile will append a unique ID for each task + if (!options::obj_path.empty()) + Filename = options::obj_path; + else if (options::TheOutputType == options::OT_SAVE_TEMPS) + Filename = output_name + ".o"; + bool SaveTemps = !Filename.empty(); + + MaxTasks = Lto->getMaxTasks(); + std::vector IsTemporary(MaxTasks); + std::vector> Filenames(MaxTasks); + + auto AddStream = [&](size_t Task) { + int FD = openOutputFile(Filename, /*TempOutFile=*/!SaveTemps, + Filenames[Task], MaxTasks > 1 ? Task : -1); + IsTemporary[Task] = !SaveTemps; - if (options::TheOutputType == options::OT_DISABLE) + return make_unique(FD, true); + }; + + check(Lto->run(AddStream)); + + if (options::TheOutputType == options::OT_DISABLE || + options::TheOutputType == options::OT_BC_ONLY) return LDPS_OK; - if (options::TheOutputType != options::OT_NORMAL) { - std::string path; - if (options::TheOutputType == options::OT_BC_ONLY) - path = output_name; - else - path = output_name + ".bc"; - saveBCFile(path, *Combined); - if (options::TheOutputType == options::OT_BC_ONLY) - return LDPS_OK; + if (options::thinlto_index_only) { + cleanup_hook(); + exit(0); } - CodeGen codeGen(std::move(Combined)); - codeGen.runAll(); + for (unsigned I = 0; I != MaxTasks; ++I) + if (!Filenames[I].empty()) + recordFile(Filenames[I].str(), IsTemporary[I]); if (!options::extra_library_path.empty() && set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK) Index: tools/llvm-lto2/CMakeLists.txt =================================================================== --- /dev/null +++ tools/llvm-lto2/CMakeLists.txt @@ -0,0 +1,11 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + LTO + Object + Support + ) + +add_llvm_tool(llvm-lto2 + llvm-lto2.cpp + ) + Index: tools/llvm-lto2/LLVMBuild.txt =================================================================== --- tools/llvm-lto2/LLVMBuild.txt +++ tools/llvm-lto2/LLVMBuild.txt @@ -1,4 +1,4 @@ -;===- ./lib/CodeGen/LLVMBuild.txt ------------------------------*- Conf -*--===; +;===- ./tools/llvm-lto/LLVMBuild.txt ----------------------------*- Conf -*--===; ; ; The LLVM Compiler Infrastructure ; @@ -15,11 +15,8 @@ ; ;===------------------------------------------------------------------------===; -[common] -subdirectories = AsmPrinter SelectionDAG MIRParser GlobalISel - [component_0] -type = Library -name = CodeGen -parent = Libraries -required_libraries = Analysis BitReader BitWriter Core Instrumentation MC ProfileData Scalar Support Target TransformUtils +type = Tool +name = llvm-lto2 +parent = Tools +required_libraries = LTO Object all-targets Index: tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- /dev/null +++ tools/llvm-lto2/llvm-lto2.cpp @@ -0,0 +1,101 @@ +#include "llvm/LTO/LTO.h" +#include "llvm/Object/IRObjectFile.h" + +using namespace llvm; +using namespace object; + +static cl::list InputFilenames(cl::Positional, cl::OneOrMore, + cl::desc("")); + +static cl::opt OutputFilename("o", cl::Required, + cl::desc("Output filename"), + cl::value_desc("filename")); + +static cl::list SymbolResolutions( + "r", + cl::desc("Specify a symbol resolution: filename,symbolname,resolution"), + cl::ZeroOrMore); + +int main(int argc, char **argv) { + cl::ParseCommandLineOptions(argc, argv, "Resolution-based LTO test harness"); + + std::map, SymbolResolution> + CommandLineResolutions; + for (std::string R : SymbolResolutions) { + StringRef Rest = R; + StringRef FileName, SymbolName; + std::tie(FileName, Rest) = Rest.split(','); + if (Rest.empty()) { + llvm::errs() << "invalid resolution: " << R << '\n'; + return 1; + } + std::tie(SymbolName, Rest) = Rest.split(','); + if (Rest.empty()) { + llvm::errs() << "invalid resolution: " << R << '\n'; + return 1; + } + SymbolResolution Res; + for (char C : Rest) { + if (C == 'p') // "prevailing" + Res.Prevailing = true; + if (C == 'l') // "local" + Res.FinalDefinitionInLinkageUnit = true; + if (C == 'x') // "externally visible" + Res.VisibleToRegularObj = true; + } + CommandLineResolutions[{FileName, SymbolName}] = Res; + } + + std::vector> MBs; + LLVMContext Ctx; + LTO Lto(Ctx); + + auto DiagHandler = [](const DiagnosticInfo &) { + exit(1); + }; + + for (std::string F : InputFilenames) { + ErrorOr> MBOrErr = MemoryBuffer::getFile(F); + if (!MBOrErr) { + errs() << argv[0] << ": " << F << ": " << MBOrErr.getError().message() + << '\n'; + return 1; + } + + ErrorOr> IROrErr = + IRObjectFile::create((*MBOrErr)->getMemBufferRef(), Ctx); + if (!IROrErr) { + errs() << argv[0] << ": " << F << ": " << IROrErr.getError().message() + << '\n'; + return 1; + } + + std::vector Res; + for (const BasicSymbolRef &Sym : LTO::symbols(IROrErr->get())) { + SmallString<64> Name; + { + raw_svector_ostream OS(Name); + Sym.printName(OS); + } + + Res.push_back(CommandLineResolutions[{F, Name.str()}]); + } + + MBs.push_back(std::move(*MBOrErr)); + Lto.add(std::move(*IROrErr), Res, DiagHandler); + } + + auto AddStream = [&](size_t Task) { + std::error_code EC; + auto S = make_unique(OutputFilename + utostr(Task), EC, + sys::fs::F_None); + if (EC) { + errs() << argv[0] << ": " << OutputFilename << Task << ": " + << EC.message() << '\n'; + exit(1); + } + return std::move(S); + }; + + Lto.run(AddStream); +}