Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -15,6 +15,7 @@ #include "llvm/LTO/ThinLTOCodeGenerator.h" #include "LLVMLTORevision.h" +#include "UpdateCompilerUsed.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" @@ -29,6 +30,13 @@ #include "llvm/IR/Mangler.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Linker/Linker.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCParser/MCAsmParser.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Object/ModuleSummaryIndexObjectFile.h" #include "llvm/Support/Debug.h" @@ -36,12 +44,14 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Path.h" #include "llvm/Support/SHA1.h" +#include "llvm/Object/RecordStreamer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/ThreadPool.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/FunctionImport.h" +#include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" @@ -294,6 +304,143 @@ PM.run(TheModule); } +// Parse inline ASM and collect the list of symbols that are not defined in +// the current module. This is inspired from IRObjectFile. +static void CollectAsmUndefinedRefs(Module &TheModule, + StringSet<> &AsmUndefinedRefs) { + const std::string &InlineAsm = TheModule.getModuleInlineAsm(); + if (InlineAsm.empty()) + return; + + Triple TT(TheModule.getTargetTriple()); + std::string Err; + const Target *T = TargetRegistry::lookupTarget(TT.str(), Err); + if (!T) + return; + + std::unique_ptr MRI(T->createMCRegInfo(TT.str())); + if (!MRI) + return; + + std::unique_ptr MAI(T->createMCAsmInfo(*MRI, TT.str())); + if (!MAI) + return; + + std::unique_ptr STI( + T->createMCSubtargetInfo(TT.str(), "", "")); + if (!STI) + return; + + std::unique_ptr MCII(T->createMCInstrInfo()); + if (!MCII) + return; + + MCObjectFileInfo MOFI; + MCContext MCCtx(MAI.get(), MRI.get(), &MOFI); + MOFI.InitMCObjectFileInfo(TT, Reloc::Default, CodeModel::Default, MCCtx); + std::unique_ptr Streamer(new RecordStreamer(MCCtx)); + T->createNullTargetStreamer(*Streamer); + + std::unique_ptr Buffer(MemoryBuffer::getMemBuffer(InlineAsm)); + SourceMgr SrcMgr; + SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc()); + std::unique_ptr Parser( + createMCAsmParser(SrcMgr, MCCtx, *Streamer, *MAI)); + + MCTargetOptions MCOptions; + std::unique_ptr TAP( + T->createMCAsmParser(*STI, *Parser, *MCII, MCOptions)); + if (!TAP) + return; + + Parser->setTargetParser(*TAP); + if (Parser->Run(false)) + return; + + for (auto &KV : *Streamer) { + StringRef Key = KV.first(); + RecordStreamer::State Value = KV.second; + switch (Value) { + case RecordStreamer::NeverSeen: + llvm_unreachable("InlineASM error"); + case RecordStreamer::DefinedGlobal: + case RecordStreamer::Defined: + break; + case RecordStreamer::Global: + case RecordStreamer::Used: + if (!TheModule.getNamedValue(Key)) + AsmUndefinedRefs.insert(Key); + break; + } + } +} + +// +static DenseSet computePreservedSymbolsForModule( + Module &TheModule, const DenseSet &GUIDPreservedSymbols, + const FunctionImporter::ExportSetTy &ExportList) { + DenseSet PreservedGV; + if (GUIDPreservedSymbols.empty()) + // Be friendly and don't nuke totally the module when the client didn't + // supply anything to preserve. + return PreservedGV; + + // Declare a callback for the internalize pass that will ask for every + // candidate GlobalValue if it can be internalized or not. + auto AddPreserveGV = [&](const GlobalValue &GV) { + auto GUID = GV.getGUID(); + if (GUIDPreservedSymbols.count(GUID) || ExportList.count(GUID)) + PreservedGV.insert(&GV); + }; + + for (auto &GV : TheModule) + AddPreserveGV(GV); + for (auto &GV : TheModule.globals()) + AddPreserveGV(GV); + for (auto &GV : TheModule.aliases()) + AddPreserveGV(GV); + + return PreservedGV; +} + +// Run internalization on \p TheModule +static void +internalizeModule(Module &TheModule, const TargetMachine &TM, + const DenseSet &PreservedGV) { + if (PreservedGV.empty()) { + // Be friendly and don't nuke totally the module when the client didn't + // supply anything to preserve. + return; + } + StringSet<> AsmUndefinedRefs; + CollectAsmUndefinedRefs(TheModule, AsmUndefinedRefs); + + // Update the llvm.compiler_used globals to force preserving libcalls and + // symbols referenced from asm + UpdateCompilerUsed(TheModule, TM, AsmUndefinedRefs); + + // Declare a callback for the internalize pass that will ask for every + // candidate GlobalValue if it can be internalized or not. + auto MustPreserveGV = + [&](const GlobalValue &GV) -> bool { return PreservedGV.count(&GV); }; + + llvm::internalizeModule(TheModule, MustPreserveGV); +} + +// Convert the PreservedSymbols map from "Name" based to "GUID" based. +static DenseSet +computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols, + const Triple &TheTriple) { + DenseSet GUIDPreservedSymbols(PreservedSymbols.size()); + for (auto &Entry : PreservedSymbols) { + StringRef Name = Entry.first(); + if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_') + Name = Name.drop_front(); + GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name)); + } + return GUIDPreservedSymbols; +} + std::unique_ptr codegenModule(Module &TheModule, TargetMachine &TM) { SmallVector OutputBuffer; @@ -378,6 +525,9 @@ sys::path::append(EntryPath, CachePath, toHex(Hasher.result())); } + // Access the path to this entry in the cache. + StringRef getEntryPath() { return EntryPath; } + // Try loading the buffer for this cache entry. ErrorOr> tryLoadingBuffer() { if (EntryPath.empty()) @@ -412,6 +562,8 @@ Module &TheModule, const ModuleSummaryIndex &Index, StringMap &ModuleMap, TargetMachine &TM, const FunctionImporter::ImportMapTy &ImportList, + const FunctionImporter::ExportSetTy &ExportList, + const DenseSet &GUIDPreservedSymbols, std::map &ResolvedODR, ThinLTOCodeGenerator::CachingOptions CacheOptions, bool DisableCodeGen, StringRef SaveTempsDir, unsigned count) { @@ -419,6 +571,13 @@ // Save temps: after IPO. saveTempBitcode(TheModule, SaveTempsDir, count, ".1.IPO.bc"); + // Prepare for internalization by computing the set of symbols to preserve. + // We need to compute the list of symbols to preserve during internalization + // before doing any promotion because after renaming we won't (easily) match + // to the original name. + auto PreservedGV = computePreservedSymbolsForModule( + TheModule, GUIDPreservedSymbols, ExportList); + // "Benchmark"-like optimization: single-source case bool SingleModule = (ModuleMap.size() == 1); @@ -432,16 +591,24 @@ // Save temps: after promotion. saveTempBitcode(TheModule, SaveTempsDir, count, ".2.promoted.bc"); + } + + // Internalization + internalizeModule(TheModule, TM, PreservedGV); + + // Save internalized bitcode + saveTempBitcode(TheModule, SaveTempsDir, count, ".3.internalized.bc"); + if (!SingleModule) { crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); // Save temps: after cross-module import. - saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc"); + saveTempBitcode(TheModule, SaveTempsDir, count, ".4.imported.bc"); } optimizeModule(TheModule, TM); - saveTempBitcode(TheModule, SaveTempsDir, count, ".3.opt.bc"); + saveTempBitcode(TheModule, SaveTempsDir, count, ".5.opt.bc"); if (DisableCodeGen) { // Configured to stop before CodeGen, serialize the bitcode and return. @@ -499,7 +666,10 @@ } void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) { - CrossReferencedSymbols.insert(Name); + // At the moment, we don't take advantage of this extra information, we're + // conservatively considering cross-references as preserved. + // CrossReferencedSymbols.insert(Name); + PreservedSymbols.insert(Name); } // TargetMachine factory @@ -676,9 +846,8 @@ // Convert the preserved symbols set from string to GUID, this is needed for // computing the caching. - DenseSet GUIDPreservedSymbols(PreservedSymbols.size()); - for (auto &Entry : PreservedSymbols) - GUIDPreservedSymbols.insert(GlobalValue::getGUID(Entry.first())); + auto GUIDPreservedSymbols = + computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple); // Parallel optimizer + codegen { @@ -690,21 +859,25 @@ auto &ExportList = ExportLists[ModuleIdentifier]; auto &DefinedFunctions = ModuleToDefinedGVSummaries[ModuleIdentifier]; + auto &ExportList = ExportLists[ModuleIdentifier]; // Resolve ODR, this has to be done early because it impacts the caching std::map ResolvedODR; - ResolveODR(*Index, ExportList, DefinedFunctions, - ModuleIdentifier, ResolvedODR); + ResolveODR(*Index, ExportList, DefinedFunctions, ModuleIdentifier, + ResolvedODR); // The module may be cached, this helps handling it. - ModuleCacheEntry CacheEntry( - CacheOptions.Path, *Index, ModuleBuffer.getBufferIdentifier(), - ImportLists[ModuleBuffer.getBufferIdentifier()], - ExportLists[ModuleBuffer.getBufferIdentifier()], ResolvedODR, - DefinedFunctions, GUIDPreservedSymbols); + ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier, + ImportLists[ModuleIdentifier], ExportList, + ResolvedODR, DefinedFunctions, + GUIDPreservedSymbols); { auto ErrOrBuffer = CacheEntry.tryLoadingBuffer(); + DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss") << " '" + << CacheEntry.getEntryPath() << "' for buffer " << count + << " " << ModuleIdentifier << "\n"); + if (ErrOrBuffer) { // Cache Hit! ProducedBinaries[count] = std::move(ErrOrBuffer.get()); @@ -719,14 +892,14 @@ auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false); // Save temps: original file. - if (!SaveTempsDir.empty()) { - saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc"); - } + saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc"); auto &ImportList = ImportLists[ModuleIdentifier]; + // Run the main process now, and generates a binary auto OutputBuffer = ProcessThinLTOModule( *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList, - ResolvedODR, CacheOptions, DisableCodeGen, SaveTempsDir, count); + ExportList, GUIDPreservedSymbols, ResolvedODR, CacheOptions, + DisableCodeGen, SaveTempsDir, count); CacheEntry.write(*OutputBuffer); ProducedBinaries[count] = std::move(OutputBuffer); Index: lib/Object/IRObjectFile.cpp =================================================================== --- lib/Object/IRObjectFile.cpp +++ lib/Object/IRObjectFile.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Object/IRObjectFile.h" -#include "RecordStreamer.h" +#include "llvm/Object/RecordStreamer.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/GVMaterializer.h" Index: lib/Object/RecordStreamer.h =================================================================== --- /dev/null +++ lib/Object/RecordStreamer.h @@ -1,42 +0,0 @@ -//===-- RecordStreamer.h - Record asm defined and used symbols ---*- C++ -*===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_OBJECT_RECORDSTREAMER_H -#define LLVM_LIB_OBJECT_RECORDSTREAMER_H - -#include "llvm/MC/MCStreamer.h" - -namespace llvm { -class RecordStreamer : public MCStreamer { -public: - enum State { NeverSeen, Global, Defined, DefinedGlobal, Used }; - -private: - StringMap Symbols; - void markDefined(const MCSymbol &Symbol); - void markGlobal(const MCSymbol &Symbol); - void markUsed(const MCSymbol &Symbol); - void visitUsedSymbol(const MCSymbol &Sym) override; - -public: - typedef StringMap::const_iterator const_iterator; - const_iterator begin(); - const_iterator end(); - RecordStreamer(MCContext &Context); - void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override; - void EmitLabel(MCSymbol *Symbol) override; - void EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) override; - bool EmitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override; - void EmitZerofill(MCSection *Section, MCSymbol *Symbol, uint64_t Size, - unsigned ByteAlignment) override; - void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, - unsigned ByteAlignment) override; -}; -} -#endif Index: lib/Object/RecordStreamer.cpp =================================================================== --- lib/Object/RecordStreamer.cpp +++ lib/Object/RecordStreamer.cpp @@ -7,7 +7,7 @@ // //===----------------------------------------------------------------------===// -#include "RecordStreamer.h" +#include "llvm/Object/RecordStreamer.h" #include "llvm/MC/MCSymbol.h" using namespace llvm;