Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -478,6 +478,11 @@ /// but if there was only one module or this was the first module we might /// not invoke mergeFrom. void removeEmptySummaryEntries(); + + /// Collect for each module the list of function it defines (GUID -> Summary). + void collectDefinedFunctionsPerModule( + StringMap> & + Module2FunctionInfoMap) const; }; } // End llvm namespace Index: include/llvm/Transforms/IPO/FunctionImport.h =================================================================== --- include/llvm/Transforms/IPO/FunctionImport.h +++ include/llvm/Transforms/IPO/FunctionImport.h @@ -19,6 +19,7 @@ namespace llvm { class LLVMContext; +class GlobalValueSummary; class Module; class ModuleSummaryIndex; @@ -59,6 +60,9 @@ /// Compute all the imports and exports for every module in the Index. /// +/// \p DefinedFunctions contains for each Module a map from (GUID) to Summary +/// for every global defined in the module. +/// /// \p ImportLists will be populated with an entry for every Module we are /// importing into. This entry is itself a map that can be passed to /// FunctionImporter::importFunctions() above (see description there). @@ -67,7 +71,9 @@ /// be imported by another module, or referenced by such a function. I.e. this /// is the set of globals that need to be promoted/renamed appropriately. void ComputeCrossModuleImport( - const ModuleSummaryIndex &Index, + const ModuleSummaryIndex &Index, + const StringMap> + &DefinedFunctions, StringMap &ImportLists, StringMap &ExportLists); } Index: lib/IR/ModuleSummaryIndex.cpp =================================================================== --- lib/IR/ModuleSummaryIndex.cpp +++ lib/IR/ModuleSummaryIndex.cpp @@ -69,6 +69,31 @@ } } +// Collect for each module the list of function it defines (GUID -> Summary). +void ModuleSummaryIndex::collectDefinedFunctionsPerModule( + StringMap> & + Module2FunctionInfoMap) const { + for (auto &GlobalList : *this) { + auto GUID = GlobalList.first; + for (auto &GlobInfo : GlobalList.second) { + auto *Summary = GlobInfo->summary(); + if (isa(Summary)) + /// Ignore global variable, focus on functions + continue; + if (auto *AS = dyn_cast(Summary)) { + if (isa(AS->getAliasee())) + /// Ignore global variable, focus on functions + continue; + } + if (auto *AS = dyn_cast(Summary)) + if (isa(&AS->getAliasee())) + /// Ignore alias to global variable, focus on functions + continue; + Module2FunctionInfoMap[Summary->modulePath()][GUID] = Summary; + } + } +} + GlobalValueInfo * ModuleSummaryIndex::getGlobalValueInfo(uint64_t ValueGUID, bool PerModuleIndex) const { Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -42,6 +42,8 @@ using namespace llvm; +#define DEBUG_TYPE "thinlto" + namespace llvm { // Flags -discard-value-names, defined in LTOCodeGenerator.cpp extern cl::opt LTODiscardValueNames; @@ -119,24 +121,15 @@ return true; } -static void ResolveODR(GlobalValue &GV, const ModuleSummaryIndex &Index, - StringRef ModulePath) { - if (GV.isDeclaration()) - return; - +static GlobalValue::LinkageTypes ResolveODR(const ModuleSummaryIndex &Index, + StringRef ModuleIdentifier, + GlobalValue::GUID GUID, + const GlobalValueSummary &GV) { auto HasMultipleCopies = [&](const GlobalValueInfoList &GVInfo) { return GVInfo.size() > 1; }; - auto getGVInfo = [&](GlobalValue &GV) -> const GlobalValueInfoList *{ - auto GUID = Function::getGlobalIdentifier(GV.getName(), GV.getLinkage(), - ModulePath); - auto It = Index.findGlobalValueInfoList(GV.getName()); - if (It == Index.end()) - return nullptr; - return &It->second; - }; - - switch (GV.getLinkage()) { + auto OriginalLinkage = GV.linkage(); + switch (OriginalLinkage) { case GlobalValue::ExternalLinkage: case GlobalValue::AvailableExternallyLinkage: case GlobalValue::AppendingLinkage: @@ -149,20 +142,19 @@ break; case GlobalValue::LinkOnceODRLinkage: case GlobalValue::WeakODRLinkage: { - auto *GVInfo = getGVInfo(GV); - if (!GVInfo) - break; + auto &GVInfo = Index.findGlobalValueInfoList(GUID)->second; // We need to emit only one of these, the first module will keep // it, but turned into a weak while the others will drop it. - if (!HasMultipleCopies(*GVInfo)) + if (!HasMultipleCopies(GVInfo)) break; - if (IsFirstDefinitionForLinker(*GVInfo, Index, ModulePath)) - GV.setLinkage(GlobalValue::WeakODRLinkage); + if (IsFirstDefinitionForLinker(GVInfo, Index, ModuleIdentifier)) + return GlobalValue::WeakODRLinkage; else - GV.setLinkage(GlobalValue::AvailableExternallyLinkage); + return GlobalValue::AvailableExternallyLinkage; break; } } + return OriginalLinkage; } /// Resolve LinkOnceODR and WeakODR. @@ -171,25 +163,50 @@ /// current module. However there is a chance that another module is still /// referencing them because of the import. We make sure we always emit at least /// one copy. -static void ResolveODR(Module &TheModule, - const ModuleSummaryIndex &Index) { +static void ResolveODR( + const ModuleSummaryIndex &Index, + const std::map &DefinedGlobals, + StringRef ModuleIdentifier, + DenseMap &ResolvedORD) { // We won't optimize the globals that are referenced by an alias for now // Ideally we should turn the alias into a global and duplicate the definition // when needed. - DenseSet GlobalInvolvedWithAlias; - for (auto &GA : TheModule.aliases()) { - auto *GO = GA.getBaseObject(); - if (auto *GV = dyn_cast(GO)) - GlobalInvolvedWithAlias.insert(GV); + DenseSet GlobalInvolvedWithAlias; + for (auto &GA : DefinedGlobals) { + if (auto AS = dyn_cast(GA.second)) + GlobalInvolvedWithAlias.insert(&AS->getAliasee()); + } + + for (auto &GV : DefinedGlobals) { + if (GlobalInvolvedWithAlias.count(GV.second)) + continue; + auto NewLinkage = ResolveODR(Index, ModuleIdentifier, GV.first, *GV.second); + if (NewLinkage != GV.second->linkage()) { + ResolvedORD[GV.first] = NewLinkage; + } } +} + +/// Fixup linkage, see ResolveODR() above. +void fixupODR( + Module &TheModule, + const DenseMap &ResolvedORD) { // Process functions and global now for (auto &GV : TheModule) { - if (!GlobalInvolvedWithAlias.count(&GV)) - ResolveODR(GV, Index, TheModule.getModuleIdentifier()); + auto NewLinkage = ResolvedORD.find(GV.getGUID()); + if (NewLinkage == ResolvedORD.end()) + continue; + DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() << "` from " + << GV.getLinkage() << " to " << NewLinkage->second << "\n"); + GV.setLinkage(NewLinkage->second); } for (auto &GV : TheModule.globals()) { - if (!GlobalInvolvedWithAlias.count(&GV)) - ResolveODR(GV, Index, TheModule.getModuleIdentifier()); + auto NewLinkage = ResolvedORD.find(GV.getGUID()); + if (NewLinkage == ResolvedORD.end()) + continue; + DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() << "` from " + << GV.getLinkage() << " to " << NewLinkage->second << "\n"); + GV.setLinkage(NewLinkage->second); } } @@ -287,13 +304,13 @@ return make_unique(std::move(OutputBuffer)); } -static std::unique_ptr -ProcessThinLTOModule(Module &TheModule, const ModuleSummaryIndex &Index, - StringMap &ModuleMap, TargetMachine &TM, - const FunctionImporter::ImportMapTy &ImportList, - ThinLTOCodeGenerator::CachingOptions CacheOptions, - bool DisableCodeGen, StringRef SaveTempsDir, - unsigned count) { +static std::unique_ptr ProcessThinLTOModule( + Module &TheModule, const ModuleSummaryIndex &Index, + StringMap &ModuleMap, TargetMachine &TM, + const FunctionImporter::ImportMapTy &ImportList, + DenseMap &ResolvedORD, + ThinLTOCodeGenerator::CachingOptions CacheOptions, bool DisableCodeGen, + StringRef SaveTempsDir, unsigned count) { // Save temps: after IPO. saveTempBitcode(TheModule, SaveTempsDir, count, ".1.IPO.bc"); @@ -307,7 +324,7 @@ // Resolve the LinkOnce/Weak ODR, trying to turn them into // "available_externally" when possible. // This is a compile-time optimization. - ResolveODR(TheModule, Index); + fixupODR(TheModule, ResolvedORD); // Save temps: after promotion. saveTempBitcode(TheModule, SaveTempsDir, count, ".2.promoted.bc"); @@ -431,10 +448,19 @@ */ void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index) { + auto ModuleIdentifier = TheModule.getModuleIdentifier(); + // Collect for each module the list of function it defines (GUID -> Summary). + StringMap> + Module2FunctionInfoMap; + Index.collectDefinedFunctionsPerModule(Module2FunctionInfoMap); // Resolve the LinkOnceODR, trying to turn them into "available_externally" // where possible. - ResolveODR(TheModule, Index); + // This is a compile-time optimization. + DenseMap ResolvedORD; + ResolveODR(Index, Module2FunctionInfoMap[ModuleIdentifier], ModuleIdentifier, + ResolvedORD); + fixupODR(TheModule, ResolvedORD); promoteModule(TheModule, Index); } @@ -445,12 +471,18 @@ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule, ModuleSummaryIndex &Index) { auto ModuleMap = generateModuleMap(Modules); + auto ModuleCount = Index.modulePaths().size(); + + // Collect for each module the list of function it defines (GUID -> Summary). + StringMap> + Module2FunctionInfoMap(ModuleCount); + Index.collectDefinedFunctionsPerModule(Module2FunctionInfoMap); // Generate import/export list - auto ModuleCount = Index.modulePaths().size(); StringMap ImportLists(ModuleCount); StringMap ExportLists(ModuleCount); - ComputeCrossModuleImport(Index, ImportLists, ExportLists); + ComputeCrossModuleImport(Index, + Module2FunctionInfoMap, ImportLists, ExportLists); auto &ImportList = ImportLists[TheModule.getModuleIdentifier()]; crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); @@ -518,11 +550,17 @@ auto ModuleMap = generateModuleMap(Modules); auto ModuleCount = Modules.size(); + // Collect for each module the list of function it defines (GUID -> Summary). + StringMap> + Module2FunctionInfoMap(ModuleCount); + Index->collectDefinedFunctionsPerModule(Module2FunctionInfoMap); + // Collect the import/export lists for all modules from the call-graph in the // combined index. StringMap ImportLists(ModuleCount); StringMap ExportLists(ModuleCount); - ComputeCrossModuleImport(*Index, ImportLists, ExportLists); + ComputeCrossModuleImport(*Index, + Module2FunctionInfoMap, ImportLists, ExportLists); // Parallel optimizer + codegen { @@ -532,6 +570,11 @@ Pool.async([&](int count) { LLVMContext Context; Context.setDiscardValueNames(LTODiscardValueNames); + auto ModuleIdentifier = ModuleBuffer.getBufferIdentifier(); + + DenseMap ResolvedORD; + ResolveODR(*Index, Module2FunctionInfoMap[ModuleIdentifier], + ModuleIdentifier, ResolvedORD); // Parse module now auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false); @@ -541,10 +584,10 @@ saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc"); } - auto &ImportList = ImportLists[TheModule->getModuleIdentifier()]; + auto &ImportList = ImportLists[ModuleIdentifier]; ProducedBinaries[count] = ProcessThinLTOModule( *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList, - CacheOptions, DisableCodeGen, SaveTempsDir, count); + ResolvedORD, CacheOptions, DisableCodeGen, SaveTempsDir, count); }, count); count++; } Index: lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- lib/Transforms/IPO/FunctionImport.cpp +++ lib/Transforms/IPO/FunctionImport.cpp @@ -252,33 +252,11 @@ /// Compute all the import and export for every module in the Index. void llvm::ComputeCrossModuleImport( - const ModuleSummaryIndex &Index, + const ModuleSummaryIndex &Index, + const StringMap> + &Module2FunctionInfoMap, StringMap &ImportLists, StringMap &ExportLists) { - auto ModuleCount = Index.modulePaths().size(); - - // Collect for each module the list of function it defines. - // GUID -> Summary - StringMap> - Module2FunctionInfoMap(ModuleCount); - - for (auto &GlobalList : Index) { - auto GUID = GlobalList.first; - for (auto &GlobInfo : GlobalList.second) { - auto *Summary = GlobInfo->summary(); - if (isa(Summary)) - /// Ignore global variable, focus on functions - continue; - if (auto *AS = dyn_cast(Summary)) - if (isa(&AS->getAliasee())) - /// Ignore alias to global variable, focus on functions - continue; - DEBUG(dbgs() << "Adding definition: Module '" << Summary->modulePath() - << "' defines '" << GUID << "'\n"); - Module2FunctionInfoMap[Summary->modulePath()][GUID] = Summary; - } - } - // For each module that has function defined, compute the import/export lists. for (auto &DefinedFunctions : Module2FunctionInfoMap) { auto &ImportsForModule = ImportLists[DefinedFunctions.first()]; @@ -488,12 +466,18 @@ Index = IndexPtr.get(); } + // Collect for each module the list of function it defines (GUID -> + // Summary). + StringMap> + Module2FunctionInfoMap; + Index->collectDefinedFunctionsPerModule(Module2FunctionInfoMap); + // First step is collecting the import/export lists // The export list is not used yet, but could limit the amount of renaming // performed in renameModuleForThinLTO() StringMap ImportLists; StringMap ExportLists; - ComputeCrossModuleImport(*Index, ImportLists, ExportLists); + ComputeCrossModuleImport(*Index, Module2FunctionInfoMap, ImportLists, ExportLists); auto &ImportList = ImportLists[M.getModuleIdentifier()]; // Next we need to promote to global scope and rename any local values that