Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -168,6 +168,12 @@ return static_cast(Flags.Linkage); } + /// Sets the linkage to the value determined by global summary-based + /// optimization. Will be applied in the ThinLTO backends. + void setLinkage(GlobalValue::LinkageTypes Linkage) { + Flags.Linkage = Linkage; + } + /// Return true if this summary is for a GlobalValue that needs promotion /// to be referenced from another module. bool needsRenaming() const { return GlobalValue::isLocalLinkage(linkage()); } @@ -446,6 +452,13 @@ return NewName.str(); } + /// Helper to obtain the unpromoted name for a global value (or the original + /// name if not promoted). + static StringRef getOriginalNameBeforePromote(StringRef Name) { + std::pair Pair = Name.split(".llvm."); + return Pair.first; + } + /// Add a new module path with the given \p Hash, mapped to the given \p /// ModID, and return an iterator to the entry in the index. ModulePathStringTableTy::iterator Index: include/llvm/LTO/LTO.h =================================================================== --- /dev/null +++ include/llvm/LTO/LTO.h @@ -0,0 +1,30 @@ +#ifndef LLVM_LTO_LTO_H +#define LLVM_LTO_LTO_H + +#include "llvm/IR/ModuleSummaryIndex.h" + +namespace llvm { + +/// Resolve Weak and LinkOnce values in the \p Index. Linkage changes recorded +/// in the index and the ThinLTO backends must apply the changes to the Module +/// via thinLTOResolveWeakForLinkerModule. +/// +/// This is done for correctness (if value exported, ensure we always +/// emit a copy), and compile-time optimization (allow drop of duplicates). +void thinLTOResolveWeakForLinkerInIndex( + ModuleSummaryIndex &Index, + std::function + isPrevailing, + std::function isExported, + std::function + recordNewLinkage); + +/// Update the linkages in the given \p Index to mark exported values +/// as external and non-exported values as internal. The ThinLTO backends +/// must apply the changes to the Module via thinLTOInternalizeModule. +void thinLTOInternalizeInIndex( + ModuleSummaryIndex &Index, + std::function isExported); +} + +#endif Index: include/llvm/Transforms/IPO/FunctionImport.h =================================================================== --- include/llvm/Transforms/IPO/FunctionImport.h +++ include/llvm/Transforms/IPO/FunctionImport.h @@ -107,6 +107,16 @@ std::error_code EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const StringMap &ImportLists); + +/// Resolve WeakForLinker values in \p TheModule based on the information +/// recorded in the summaries during global summary-based analysis. +void thinLTOResolveWeakForLinkerModule(Module &TheModule, + const GVSummaryMapTy &DefinedGlobals); + +/// Internalize \p TheModule based on the information recorded in the summaries +/// during global summary-based analysis. +void thinLTOInternalizeModule(Module &TheModule, + const GVSummaryMapTy &DefinedGlobals); } #endif // LLVM_FUNCTIONIMPORT_H Index: lib/LTO/CMakeLists.txt =================================================================== --- lib/LTO/CMakeLists.txt +++ lib/LTO/CMakeLists.txt @@ -48,6 +48,7 @@ add_llvm_library(LLVMLTO + LTO.cpp LTOModule.cpp LTOCodeGenerator.cpp UpdateCompilerUsed.cpp Index: lib/LTO/LTO.cpp =================================================================== --- /dev/null +++ lib/LTO/LTO.cpp @@ -0,0 +1,106 @@ +#include "llvm/LTO/LTO.h" + +namespace llvm { + +static void thinLTOResolveWeakForLinkerGUID( + GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID, + DenseSet &GlobalInvolvedWithAlias, + std::function + isPrevailing, + std::function isExported, + std::function + recordNewLinkage) { + auto HasMultipleCopies = GVSummaryList.size() > 1; + + for (auto &S : GVSummaryList) { + if (GlobalInvolvedWithAlias.count(S.get())) + continue; + GlobalValue::LinkageTypes OriginalLinkage = S->linkage(); + if (!GlobalValue::isWeakForLinker(OriginalLinkage)) + continue; + // We need to emit only one of these, the first module will keep it, + // but turned into a weak, while the others will drop it when possible. + if (!HasMultipleCopies) { + // Exported Linkonce needs to be promoted to not be discarded. + // FIXME: This should handle LinkOnceAny as well, but that should be a + // follow-on to the NFC restructuring: + // if (GlobalValue::isLinkOnceLinkage(OriginalLinkage) && + // isExported(S->modulePath(), GUID)) + // S->setLinkage(GlobalValue::getWeakLinkage( + // GlobalValue::isLinkOnceODRLinkage(OriginalLinkage))); + if (GlobalValue::isLinkOnceODRLinkage(OriginalLinkage) && + isExported(S->modulePath(), GUID)) + S->setLinkage(GlobalValue::WeakODRLinkage); + } else if (isPrevailing(GUID, S.get())) { + // FIXME: This should handle LinkOnceAny as well, but that should be a + // follow-on to the NFC restructuring: + // if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) + // S->setLinkage(GlobalValue::getWeakLinkage( + // GlobalValue::isLinkOnceODRLinkage(OriginalLinkage))); + if (GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)) + S->setLinkage(GlobalValue::WeakODRLinkage); + } + // Alias can't be turned into available_externally. + else if (!isa(S.get()) && + (GlobalValue::isLinkOnceODRLinkage(OriginalLinkage) || + GlobalValue::isWeakODRLinkage(OriginalLinkage))) + S->setLinkage(GlobalValue::AvailableExternallyLinkage); + if (S->linkage() != OriginalLinkage) + recordNewLinkage(S->modulePath(), GUID, S->linkage()); + } +} + +// Resolve Weak and LinkOnce values in the \p Index. +// +// We'd like to drop these function if they are no longer referenced in the +// current module. However there is a chance that another module is still +// referencing them because of the import. We make sure we always emit at least +// one copy. +void thinLTOResolveWeakForLinkerInIndex( + ModuleSummaryIndex &Index, + std::function + isPrevailing, + std::function isExported, + std::function + recordNewLinkage) { + if (Index.modulePaths().size() == 1) + // Nothing to do if we don't have multiple modules + return; + + // We won't optimize the globals that are referenced by an alias for now + // Ideally we should turn the alias into a global and duplicate the definition + // when needed. + DenseSet GlobalInvolvedWithAlias; + for (auto &I : Index) + for (auto &S : I.second) + if (auto AS = dyn_cast(S.get())) + GlobalInvolvedWithAlias.insert(&AS->getAliasee()); + + for (auto &I : Index) + thinLTOResolveWeakForLinkerGUID(I.second, I.first, GlobalInvolvedWithAlias, + isPrevailing, isExported, recordNewLinkage); +} + +static void thinLTOInternalizeGUID( + GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID, + std::function isExported) { + for (auto &S : GVSummaryList) { + if (isExported(S->modulePath(), GUID)) { + if (GlobalValue::isLocalLinkage(S->linkage())) + S->setLinkage(GlobalValue::ExternalLinkage); + } else { + if (GlobalValue::isExternalLinkage(S->linkage())) + S->setLinkage(GlobalValue::InternalLinkage); + } + } +} + +// Update the linkages in the given \p Index to mark exported values +// as external and non-exported values as internal. +void thinLTOInternalizeInIndex( + ModuleSummaryIndex &Index, + std::function isExported) { + for (auto &I : Index) + thinLTOInternalizeGUID(I.second, I.first, isExported); +} +} Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -32,11 +32,10 @@ #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Mangler.h" #include "llvm/IRReader/IRReader.h" +#include "llvm/LTO/LTO.h" #include "llvm/Linker/Linker.h" #include "llvm/MC/SubtargetFeature.h" -#include "llvm/Object/IRObjectFile.h" #include "llvm/Object/ModuleSummaryIndexObjectFile.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/CachePruning.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Path.h" @@ -47,7 +46,6 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/FunctionImport.h" -#include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" @@ -109,9 +107,17 @@ WriteBitcodeToFile(&TheModule, OS, /* ShouldPreserveUseListOrder */ true); } -bool IsFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList, - const ModuleSummaryIndex &Index, - StringRef ModulePath) { +static const GlobalValueSummary * +getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { + // If there is any strong definition anywhere, get it. + auto StrongDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage) && + !GlobalValue::isWeakForLinker(Linkage); + }); + if (StrongDefForLinker != GVSummaryList.end()) + return StrongDefForLinker->get(); // Get the first *linker visible* definition for this global in the summary // list. auto FirstDefForLinker = llvm::find_if( @@ -119,130 +125,23 @@ auto Linkage = Summary->linkage(); return !GlobalValue::isAvailableExternallyLinkage(Linkage); }); - // If \p GV is not the first definition, give up... - if ((*FirstDefForLinker)->modulePath() != ModulePath) - return false; - // If there is any strong definition anywhere, do not bother emitting this. - if (llvm::any_of( - GVSummaryList, - [](const std::unique_ptr &Summary) { - auto Linkage = Summary->linkage(); - return !GlobalValue::isAvailableExternallyLinkage(Linkage) && - !GlobalValue::isWeakForLinker(Linkage); - })) - return false; - return true; + assert(FirstDefForLinker != GVSummaryList.end()); + return FirstDefForLinker->get(); } -static GlobalValue::LinkageTypes -ResolveODR(const ModuleSummaryIndex &Index, - const FunctionImporter::ExportSetTy &ExportList, - const DenseSet &GUIDPreservedSymbols, - StringRef ModuleIdentifier, GlobalValue::GUID GUID, - const GlobalValueSummary &GV) { +// Populate map of GUID to the prevailing copy for any multiply defined +// symbols. Currently assume first copy is prevailing, or any strong +// definition. Can be refined with Linker information in the future. +static void computePrevailingCopies( + const ModuleSummaryIndex &Index, + DenseMap &PrevailingCopy) { auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) { return GVSummaryList.size() > 1; }; - auto OriginalLinkage = GV.linkage(); - switch (OriginalLinkage) { - case GlobalValue::ExternalLinkage: - case GlobalValue::AvailableExternallyLinkage: - case GlobalValue::AppendingLinkage: - case GlobalValue::InternalLinkage: - case GlobalValue::PrivateLinkage: - case GlobalValue::ExternalWeakLinkage: - case GlobalValue::CommonLinkage: - case GlobalValue::LinkOnceAnyLinkage: - case GlobalValue::WeakAnyLinkage: - break; - case GlobalValue::LinkOnceODRLinkage: - case GlobalValue::WeakODRLinkage: { - auto &GVSummaryList = Index.findGlobalValueSummaryList(GUID)->second; - // We need to emit only one of these, the first module will keep - // it, but turned into a weak while the others will drop it. - if (!HasMultipleCopies(GVSummaryList)) { - // Exported LinkonceODR needs to be promoted to not be discarded - if (GlobalValue::isDiscardableIfUnused(OriginalLinkage) && - (ExportList.count(GUID) || GUIDPreservedSymbols.count(GUID))) - return GlobalValue::WeakODRLinkage; - break; - } - if (IsFirstDefinitionForLinker(GVSummaryList, Index, ModuleIdentifier)) - return GlobalValue::WeakODRLinkage; - else if (isa(&GV)) - // Alias can't be turned into available_externally. - return OriginalLinkage; - return GlobalValue::AvailableExternallyLinkage; - } - } - return OriginalLinkage; -} - -/// Resolve LinkOnceODR and WeakODR. -/// -/// We'd like to drop these function if they are no longer referenced in the -/// current module. However there is a chance that another module is still -/// referencing them because of the import. We make sure we always emit at least -/// one copy. -static void ResolveODR( - const ModuleSummaryIndex &Index, - const FunctionImporter::ExportSetTy &ExportList, - const DenseSet &GUIDPreservedSymbols, - const GVSummaryMapTy &DefinedGlobals, StringRef ModuleIdentifier, - std::map &ResolvedODR) { - if (Index.modulePaths().size() == 1) - // Nothing to do if we don't have multiple modules - return; - - // We won't optimize the globals that are referenced by an alias for now - // Ideally we should turn the alias into a global and duplicate the definition - // when needed. - DenseSet GlobalInvolvedWithAlias; - for (auto &GA : DefinedGlobals) { - if (auto AS = dyn_cast(GA.second)) - GlobalInvolvedWithAlias.insert(&AS->getAliasee()); - } - - for (auto &GV : DefinedGlobals) { - if (GlobalInvolvedWithAlias.count(GV.second)) - continue; - auto NewLinkage = - ResolveODR(Index, ExportList, GUIDPreservedSymbols, ModuleIdentifier, GV.first, *GV.second); - if (NewLinkage != GV.second->linkage()) { - ResolvedODR[GV.first] = NewLinkage; - } - } -} - -/// Fixup linkage, see ResolveODR() above. -void fixupODR( - Module &TheModule, - const std::map &ResolvedODR) { - // Process functions and global now - for (auto &GV : TheModule) { - auto NewLinkage = ResolvedODR.find(GV.getGUID()); - if (NewLinkage == ResolvedODR.end()) - continue; - DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() << "` from " - << GV.getLinkage() << " to " << NewLinkage->second << "\n"); - GV.setLinkage(NewLinkage->second); - } - for (auto &GV : TheModule.globals()) { - auto NewLinkage = ResolvedODR.find(GV.getGUID()); - if (NewLinkage == ResolvedODR.end()) - continue; - DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() << "` from " - << GV.getLinkage() << " to " << NewLinkage->second << "\n"); - GV.setLinkage(NewLinkage->second); - } - for (auto &GV : TheModule.aliases()) { - auto NewLinkage = ResolvedODR.find(GV.getGUID()); - if (NewLinkage == ResolvedODR.end()) - continue; - DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() << "` from " - << GV.getLinkage() << " to " << NewLinkage->second << "\n"); - GV.setLinkage(NewLinkage->second); + for (auto &I : Index) { + if (HasMultipleCopies(I.second)) + PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second); } } @@ -316,63 +215,6 @@ PM.run(TheModule); } -// Create a DenseSet of GlobalValue to be used with the Internalizer. -static DenseSet computePreservedSymbolsForModule( - Module &TheModule, const DenseSet &GUIDPreservedSymbols, - const FunctionImporter::ExportSetTy &ExportList) { - DenseSet PreservedGV; - if (GUIDPreservedSymbols.empty()) - // Early exit: internalize is disabled when there is nothing to preserve. - return PreservedGV; - - auto AddPreserveGV = [&](const GlobalValue &GV) { - auto GUID = GV.getGUID(); - if (GUIDPreservedSymbols.count(GUID) || ExportList.count(GUID)) - PreservedGV.insert(&GV); - }; - - for (auto &GV : TheModule) - AddPreserveGV(GV); - for (auto &GV : TheModule.globals()) - AddPreserveGV(GV); - for (auto &GV : TheModule.aliases()) - AddPreserveGV(GV); - - return PreservedGV; -} - -// Run internalization on \p TheModule -static void -doInternalizeModule(Module &TheModule, const TargetMachine &TM, - const DenseSet &PreservedGV) { - if (PreservedGV.empty()) { - // Be friendly and don't nuke totally the module when the client didn't - // supply anything to preserve. - return; - } - - // Parse inline ASM and collect the list of symbols that are not defined in - // the current module. - StringSet<> AsmUndefinedRefs; - object::IRObjectFile::CollectAsmUndefinedRefs( - Triple(TheModule.getTargetTriple()), TheModule.getModuleInlineAsm(), - [&AsmUndefinedRefs](StringRef Name, object::BasicSymbolRef::Flags Flags) { - if (Flags & object::BasicSymbolRef::SF_Undefined) - AsmUndefinedRefs.insert(Name); - }); - - // Update the llvm.compiler_used globals to force preserving libcalls and - // symbols referenced from asm - UpdateCompilerUsed(TheModule, TM, AsmUndefinedRefs); - - // Declare a callback for the internalize pass that will ask for every - // candidate GlobalValue if it can be internalized or not. - auto MustPreserveGV = - [&](const GlobalValue &GV) -> bool { return PreservedGV.count(&GV); }; - - llvm::internalizeModule(TheModule, MustPreserveGV); -} - // Convert the PreservedSymbols map from "Name" based to "GUID" based. static DenseSet computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols, @@ -506,22 +348,25 @@ } }; -static std::unique_ptr ProcessThinLTOModule( - Module &TheModule, const ModuleSummaryIndex &Index, - StringMap &ModuleMap, TargetMachine &TM, - const FunctionImporter::ImportMapTy &ImportList, - const FunctionImporter::ExportSetTy &ExportList, - const DenseSet &GUIDPreservedSymbols, - std::map &ResolvedODR, - ThinLTOCodeGenerator::CachingOptions CacheOptions, bool DisableCodeGen, - StringRef SaveTempsDir, unsigned count) { - - // Prepare for internalization by computing the set of symbols to preserve. - // We need to compute the list of symbols to preserve during internalization - // before doing any promotion because after renaming we won't (easily) match - // to the original name. - auto PreservedGV = computePreservedSymbolsForModule( - TheModule, GUIDPreservedSymbols, ExportList); +static std::unique_ptr +ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index, + StringMap &ModuleMap, TargetMachine &TM, + const FunctionImporter::ImportMapTy &ImportList, + const FunctionImporter::ExportSetTy &ExportList, + const DenseSet &GUIDPreservedSymbols, + const GVSummaryMapTy &DefinedGlobals, + ThinLTOCodeGenerator::CachingOptions CacheOptions, + bool DisableCodeGen, StringRef SaveTempsDir, + unsigned count) { + + auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { + return ExportList.count(GUID) || GUIDPreservedSymbols.count(GUID); + }; + + // Use global summary-based analysis to identify symbols that can be + // internalized (because they aren't exported or preserved as per callback). + // Changes are made in the index, consumed in the ThinLTO backends. + thinLTOInternalizeInIndex(Index, isExported); // "Benchmark"-like optimization: single-source case bool SingleModule = (ModuleMap.size() == 1); @@ -529,17 +374,19 @@ if (!SingleModule) { promoteModule(TheModule, Index); - // Resolve the LinkOnce/Weak ODR, trying to turn them into - // "available_externally" when possible. - // This is a compile-time optimization. - fixupODR(TheModule, ResolvedODR); + // Apply summary-based LinkOnce/Weak resolution decisions. + thinLTOResolveWeakForLinkerModule(TheModule, DefinedGlobals); // Save temps: after promotion. saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc"); } - // Internalization - doInternalizeModule(TheModule, TM, PreservedGV); + // Be friendly and don't nuke totally the module when the client didn't + // supply anything to preserve. + if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) { + // Apply summary-based internalization decisions. + thinLTOInternalizeModule(TheModule, DefinedGlobals); + } // Save internalized bitcode saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc"); @@ -678,21 +525,48 @@ StringMap ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); - auto &ExportList = ExportLists[ModuleIdentifier]; // Convert the preserved symbols set from string to GUID auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple); - // Resolve the LinkOnceODR, trying to turn them into "available_externally" - // where possible. - // This is a compile-time optimization. + DenseMap PrevailingCopy; + computePrevailingCopies(Index, PrevailingCopy); + + // Resolve LinkOnce/Weak symbols, this has to be computed early because it + // impacts the caching. Besides recording in the ResolvedODR map for caching, + // the changes are recorded in the index for application during the ThinLTO + // backends. This is needed for correctness for exported symbols (ensure + // at least one copy kept) and a compile-time optimization (to drop duplicate + // copies when possible). + + auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { + const auto &Prevailing = PrevailingCopy.find(GUID); + // Not in map means that there was only one copy, which must be prevailing. + if (Prevailing == PrevailingCopy.end()) + return true; + return Prevailing->second == S; + }; + + auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { + auto &ExportList = ExportLists[ModuleIdentifier]; + return ExportList.count(GUID) || GUIDPreservedSymbols.count(GUID); + }; + // We use a std::map here to be able to have a defined ordering when // producing a hash for the cache entry. - std::map ResolvedODR; - ResolveODR(Index, ExportList, GUIDPreservedSymbols, ModuleToDefinedGVSummaries[ModuleIdentifier], - ModuleIdentifier, ResolvedODR); - fixupODR(TheModule, ResolvedODR); + StringMap> ResolvedODR; + auto recordNewLinkage = [&](StringRef ModuleIdentifier, + GlobalValue::GUID GUID, + GlobalValue::LinkageTypes NewLinkage) { + ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; + }; + + thinLTOResolveWeakForLinkerInIndex(Index, isPrevailing, isExported, + recordNewLinkage); + + thinLTOResolveWeakForLinkerModule( + TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]); promoteModule(TheModule, Index); } @@ -791,9 +665,16 @@ auto &ExportList = ExportLists[ModuleIdentifier]; // Internalization - auto PreservedGV = computePreservedSymbolsForModule( - TheModule, GUIDPreservedSymbols, ExportList); - doInternalizeModule(TheModule, *TMBuilder.create(), PreservedGV); + auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { + return ExportList.count(GUID) || GUIDPreservedSymbols.count(GUID); + }; + thinLTOInternalizeInIndex(Index, isExported); + // Be friendly and don't nuke totally the module when the client didn't + // supply anything to preserve. + if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) { + thinLTOInternalizeModule(TheModule, + ModuleToDefinedGVSummaries[ModuleIdentifier]); + } } /** @@ -864,6 +745,9 @@ StringMap ModuleToDefinedGVSummaries(ModuleCount); Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); + DenseMap PrevailingCopy; + computePrevailingCopies(*Index, PrevailingCopy); + // Collect the import/export lists for all modules from the call-graph in the // combined index. StringMap ImportLists(ModuleCount); @@ -876,6 +760,37 @@ auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple); + auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { + const auto &Prevailing = PrevailingCopy.find(GUID); + // Not in map means that there was only one copy, which must be prevailing. + if (Prevailing == PrevailingCopy.end()) + return true; + return Prevailing->second == S; + }; + + auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { + auto &ExportList = ExportLists[ModuleIdentifier]; + return ExportList.count(GUID) || GUIDPreservedSymbols.count(GUID); + }; + + // We use a std::map here to be able to have a defined ordering when + // producing a hash for the cache entry. + StringMap> ResolvedODR; + auto recordNewLinkage = [&](StringRef ModuleIdentifier, + GlobalValue::GUID GUID, + GlobalValue::LinkageTypes NewLinkage) { + ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; + }; + + // Resolve LinkOnce/Weak symbols, this has to be computed early because it + // impacts the caching. Besides recording in the ResolvedODR map for caching, + // the changes are recorded in the index for application during the ThinLTO + // backends. This is needed for correctness for exported symbols (ensure + // at least one copy kept) and a compile-time optimization (to drop duplicate + // copies when possible). + thinLTOResolveWeakForLinkerInIndex(*Index, isPrevailing, isExported, + recordNewLinkage); + // Parallel optimizer + codegen { ThreadPool Pool(ThreadCount); @@ -887,18 +802,11 @@ auto &DefinedFunctions = ModuleToDefinedGVSummaries[ModuleIdentifier]; - // Resolve ODR, this has to be done early because it impacts the caching - // We use a std::map here to be able to have a defined ordering when - // producing a hash for the cache entry. - std::map ResolvedODR; - ResolveODR(*Index, ExportList, GUIDPreservedSymbols, DefinedFunctions, ModuleIdentifier, - ResolvedODR); - // The module may be cached, this helps handling it. ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier, ImportLists[ModuleIdentifier], ExportList, - ResolvedODR, DefinedFunctions, - GUIDPreservedSymbols); + ResolvedODR[ModuleIdentifier], + DefinedFunctions, GUIDPreservedSymbols); { auto ErrOrBuffer = CacheEntry.tryLoadingBuffer(); @@ -927,7 +835,8 @@ // Run the main process now, and generates a binary auto OutputBuffer = ProcessThinLTOModule( *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList, - ExportList, GUIDPreservedSymbols, ResolvedODR, CacheOptions, + ExportList, GUIDPreservedSymbols, + ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions, DisableCodeGen, SaveTempsDir, count); CacheEntry.write(*OutputBuffer); Index: lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- lib/Transforms/IPO/FunctionImport.cpp +++ lib/Transforms/IPO/FunctionImport.cpp @@ -15,17 +15,21 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringSet.h" +#include "llvm/ADT/Triple.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Linker/Linker.h" +#include "llvm/Object/IRObjectFile.h" #include "llvm/Object/ModuleSummaryIndexObjectFile.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #define DEBUG_TYPE "function-import" @@ -460,6 +464,78 @@ return std::error_code(); } +/// Fixup WeakForLinker linkages in \p TheModule based on summary analysis. +void llvm::thinLTOResolveWeakForLinkerModule( + Module &TheModule, const GVSummaryMapTy &DefinedGlobals) { + auto updateLinkage = [&](GlobalValue &GV) { + if (!GlobalValue::isWeakForLinker(GV.getLinkage())) + return; + // See if the global summary analysis computed a new resolved linkage. + const auto &GS = DefinedGlobals.find(GV.getGUID()); + if (GS == DefinedGlobals.end()) + return; + auto NewLinkage = GS->second->linkage(); + if (NewLinkage == GV.getLinkage()) + return; + DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() << "` from " + << GV.getLinkage() << " to " << NewLinkage << "\n"); + GV.setLinkage(NewLinkage); + }; + + // Process functions and global now + for (auto &GV : TheModule) + updateLinkage(GV); + for (auto &GV : TheModule.globals()) + updateLinkage(GV); + for (auto &GV : TheModule.aliases()) + updateLinkage(GV); +} + +// Run internalization on \p TheModule based on symmary analysis. +void llvm::thinLTOInternalizeModule(Module &TheModule, + const GVSummaryMapTy &DefinedGlobals) { + // Parse inline ASM and collect the list of symbols that are not defined in + // the current module. + StringSet<> AsmUndefinedRefs; + object::IRObjectFile::CollectAsmUndefinedRefs( + Triple(TheModule.getTargetTriple()), TheModule.getModuleInlineAsm(), + [&AsmUndefinedRefs](StringRef Name, object::BasicSymbolRef::Flags Flags) { + if (Flags & object::BasicSymbolRef::SF_Undefined) + AsmUndefinedRefs.insert(Name); + }); + + // Declare a callback for the internalize pass that will ask for every + // candidate GlobalValue if it can be internalized or not. + auto MustPreserveGV = [&](const GlobalValue &GV) -> bool { + // Can't be internalized if referenced in inline asm. + if (AsmUndefinedRefs.count(GV.getName())) + return true; + + // Lookup the linkage recorded in the summaries during global analysis. + const auto &GS = DefinedGlobals.find(GV.getGUID()); + GlobalValue::LinkageTypes Linkage; + if (GS == DefinedGlobals.end()) { + // Must have been promoted (possibly conservatively). Find original + // name so that we can access the correct summary and see if it can + // be internalized again. + // FIXME: Eventually we should control promotion instead of promoting + // and internalizing again. + StringRef OrigName = + ModuleSummaryIndex::getOriginalNameBeforePromote(GV.getName()); + std::string OrigId = GlobalValue::getGlobalIdentifier( + OrigName, GlobalValue::InternalLinkage, + TheModule.getSourceFileName()); + const auto &GS = DefinedGlobals.find(GlobalValue::getGUID(OrigId)); + assert(GS != DefinedGlobals.end()); + Linkage = GS->second->linkage(); + } else + Linkage = GS->second->linkage(); + return Linkage == GlobalValue::ExternalLinkage; + }; + + llvm::internalizeModule(TheModule, MustPreserveGV); +} + // Automatically import functions in Module \p DestModule based on the summaries // index. //