Index: include/llvm/Analysis/ModuleSummaryAnalysis.h =================================================================== --- include/llvm/Analysis/ModuleSummaryAnalysis.h +++ include/llvm/Analysis/ModuleSummaryAnalysis.h @@ -29,7 +29,7 @@ /// BlockFrequencyInfo for a given function, that can be provided via /// a std::function callback. Otherwise, this routine will manually construct /// that information. -ModuleSummaryIndex buildModuleSummaryIndex( +ModuleSummaryIndex *buildModuleSummaryIndex( const Module &M, std::function GetBFICallback, ProfileSummaryInfo *PSI); @@ -41,14 +41,14 @@ static AnalysisKey Key; public: - typedef ModuleSummaryIndex Result; + typedef std::unique_ptr Result; Result run(Module &M, ModuleAnalysisManager &AM); }; /// Legacy wrapper pass to provide the ModuleSummaryIndex object. class ModuleSummaryIndexWrapperPass : public ModulePass { - Optional Index; + std::unique_ptr Index; public: static char ID; Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -79,44 +79,53 @@ GlobalValueSummaryList SummaryList; }; -/// Map from global value GUID to corresponding summary structures. Use a -/// std::map rather than a DenseMap so that pointers to the map's value_type -/// (which are used by ValueInfo) are not invalidated by insertion. Also it will -/// likely incur less overhead, as the value type is not very small and the size -/// of the map is unknown, resulting in inefficiencies due to repeated -/// insertions and resizing. +/// Map from global value GUID to corresponding summary structures. using GlobalValueSummaryMapTy = - std::map; + DenseMap; /// Struct that holds a reference to a particular GUID in a global value /// summary. struct ValueInfo { - const GlobalValueSummaryMapTy::value_type *Ref = nullptr; + GlobalValue::GUID GUID = 0; + const GlobalValueSummaryMapTy *Map = nullptr; ValueInfo() = default; - ValueInfo(const GlobalValueSummaryMapTy::value_type *Ref) : Ref(Ref) {} + ValueInfo(GlobalValue::GUID GUID, const GlobalValueSummaryMapTy *Map) + : GUID(GUID), Map(Map) {} - operator bool() const { return Ref; } + operator bool() const { return Map; } - GlobalValue::GUID getGUID() const { return Ref->first; } - const GlobalValue *getValue() const { return Ref->second.GV; } + GlobalValue::GUID getGUID() const { return GUID; } + const GlobalValue *getValue() const { + if (Map == nullptr) + return nullptr; + auto I = Map->find(GUID); + return I == Map->end() ? nullptr : I->second.GV; + } ArrayRef> getSummaryList() const { - return Ref->second.SummaryList; + return Map->find(GUID)->second.SummaryList; + } + + void addSummary(std::unique_ptr Summary) { + const_cast(Map->find(GUID)->second.SummaryList) + .push_back(std::move(Summary)); } }; template <> struct DenseMapInfo { static inline ValueInfo getEmptyKey() { - return ValueInfo((GlobalValueSummaryMapTy::value_type *)-1); + return ValueInfo(-1, (const GlobalValueSummaryMapTy *)-1); } static inline ValueInfo getTombstoneKey() { - return ValueInfo((GlobalValueSummaryMapTy::value_type *)-2); + return ValueInfo(-2, (const GlobalValueSummaryMapTy *)-2); } - static bool isEqual(ValueInfo L, ValueInfo R) { return L.Ref == R.Ref; } - static unsigned getHashValue(ValueInfo I) { return (uintptr_t)I.Ref; } + static bool isEqual(ValueInfo L, ValueInfo R) { + return L.GUID == R.GUID && L.Map == R.Map; + } + static unsigned getHashValue(ValueInfo I) { return I.GUID; } }; /// \brief Function and variable summary information to aid decisions and @@ -542,7 +551,7 @@ /// Mapping from original ID to GUID. If original ID can map to multiple /// GUIDs, it will be mapped to 0. - std::map OidGuidMap; + DenseMap OidGuidMap; /// Indicates that summary-based GlobalValue GC has run, and values with /// GVFlags::Live==false are really dead. Otherwise, all values must be @@ -557,7 +566,9 @@ GlobalValueSummaryMapTy::value_type * getOrInsertValuePtr(GlobalValue::GUID GUID) { - return &*GlobalValueMap.emplace(GUID, GlobalValueSummaryInfo{}).first; + return &*GlobalValueMap + .insert(std::make_pair(GUID, GlobalValueSummaryInfo{})) + .first; } public: @@ -582,19 +593,21 @@ /// Return a ValueInfo for GUID if it exists, otherwise return ValueInfo(). ValueInfo getValueInfo(GlobalValue::GUID GUID) const { auto I = GlobalValueMap.find(GUID); - return ValueInfo(I == GlobalValueMap.end() ? nullptr : &*I); + return ValueInfo(GUID, + I == GlobalValueMap.end() ? nullptr : &GlobalValueMap); } /// Return a ValueInfo for \p GUID. ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID) { - return ValueInfo(getOrInsertValuePtr(GUID)); + getOrInsertValuePtr(GUID); + return ValueInfo(GUID, &GlobalValueMap); } /// Return a ValueInfo for \p GV and mark it as belonging to GV. ValueInfo getOrInsertValueInfo(const GlobalValue *GV) { auto VP = getOrInsertValuePtr(GV->getGUID()); VP->second.GV = GV; - return ValueInfo(VP); + return ValueInfo(GV->getGUID(), &GlobalValueMap); } /// Return the GUID for \p OriginalId in the OidGuidMap. @@ -620,10 +633,7 @@ void addGlobalValueSummary(ValueInfo VI, std::unique_ptr Summary) { addOriginalName(VI.getGUID(), Summary->getOriginalName()); - // Here we have a notionally const VI, but the value it points to is owned - // by the non-const *this. - const_cast(VI.Ref) - ->second.SummaryList.push_back(std::move(Summary)); + VI.addSummary(std::move(Summary)); } /// Add an original name for the value of the given GUID. Index: lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- lib/Analysis/ModuleSummaryAnalysis.cpp +++ lib/Analysis/ModuleSummaryAnalysis.cpp @@ -326,12 +326,12 @@ Summary->setLive(true); } -ModuleSummaryIndex llvm::buildModuleSummaryIndex( +ModuleSummaryIndex *llvm::buildModuleSummaryIndex( const Module &M, std::function GetBFICallback, ProfileSummaryInfo *PSI) { assert(PSI); - ModuleSummaryIndex Index; + ModuleSummaryIndex *Index = new ModuleSummaryIndex(); // Identify the local values in the llvm.used and llvm.compiler.used sets, // which should not be exported as they would then require renaming and @@ -369,7 +369,7 @@ BFI = BFIPtr.get(); } - computeFunctionSummary(Index, M, F, BFI, PSI, !LocalsUsed.empty(), + computeFunctionSummary(*Index, M, F, BFI, PSI, !LocalsUsed.empty(), CantBePromoted); } @@ -378,16 +378,16 @@ for (const GlobalVariable &G : M.globals()) { if (G.isDeclaration()) continue; - computeVariableSummary(Index, G, CantBePromoted); + computeVariableSummary(*Index, G, CantBePromoted); } // Compute summaries for all aliases defined in module, and save in the // index. for (const GlobalAlias &A : M.aliases()) - computeAliasSummary(Index, A, CantBePromoted); + computeAliasSummary(*Index, A, CantBePromoted); for (auto *V : LocalsUsed) { - auto *Summary = Index.getGlobalValueSummary(*V); + auto *Summary = Index->getGlobalValueSummary(*V); assert(Summary && "Missing summary for global value"); Summary->setNotEligibleToImport(); } @@ -395,11 +395,11 @@ // The linker doesn't know about these LLVM produced values, so we need // to flag them as live in the index to ensure index-based dead value // analysis treats them as live roots of the analysis. - setLiveRoot(Index, "llvm.used"); - setLiveRoot(Index, "llvm.compiler.used"); - setLiveRoot(Index, "llvm.global_ctors"); - setLiveRoot(Index, "llvm.global_dtors"); - setLiveRoot(Index, "llvm.global.annotations"); + setLiveRoot(*Index, "llvm.used"); + setLiveRoot(*Index, "llvm.compiler.used"); + setLiveRoot(*Index, "llvm.global_ctors"); + setLiveRoot(*Index, "llvm.global_dtors"); + setLiveRoot(*Index, "llvm.global.annotations"); if (!M.getModuleInlineAsm().empty()) { // Collect the local values defined by module level asm, and set up @@ -412,7 +412,7 @@ // be listed on the llvm.used or llvm.compiler.used global and marked as // referenced from there. ModuleSymbolTable::CollectAsmSymbols( - M, [&M, &Index, &CantBePromoted](StringRef Name, + M, [&M, Index, &CantBePromoted](StringRef Name, object::BasicSymbolRef::Flags Flags) { // Symbols not marked as Weak or Global are local definitions. if (Flags & (object::BasicSymbolRef::SF_Weak | @@ -437,12 +437,12 @@ ArrayRef{}, ArrayRef{}, ArrayRef{}); - Index.addGlobalValueSummary(Name, std::move(Summary)); + Index->addGlobalValueSummary(Name, std::move(Summary)); } else { std::unique_ptr Summary = llvm::make_unique(GVFlags, ArrayRef{}); - Index.addGlobalValueSummary(Name, std::move(Summary)); + Index->addGlobalValueSummary(Name, std::move(Summary)); } }); } @@ -452,7 +452,7 @@ mdconst::extract_or_null(M.getModuleFlag("ThinLTO"))) IsThinLTO = MD->getZExtValue(); - for (auto &GlobalList : Index) { + for (auto &GlobalList : *Index) { // Ignore entries for references that are undefined in the current module. if (GlobalList.second.SummaryList.empty()) continue; @@ -489,17 +489,17 @@ AnalysisKey ModuleSummaryIndexAnalysis::Key; -ModuleSummaryIndex +std::unique_ptr ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) { ProfileSummaryInfo &PSI = AM.getResult(M); auto &FAM = AM.getResult(M).getManager(); - return buildModuleSummaryIndex( + return std::unique_ptr(buildModuleSummaryIndex( M, [&FAM](const Function &F) { return &FAM.getResult( *const_cast(&F)); }, - &PSI); + &PSI)); } char ModuleSummaryIndexWrapperPass::ID = 0; @@ -521,19 +521,18 @@ bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) { auto &PSI = *getAnalysis().getPSI(); - Index = buildModuleSummaryIndex( + Index.reset(buildModuleSummaryIndex( M, [this](const Function &F) { return &(this->getAnalysis( *const_cast(&F)) .getBFI()); }, - &PSI); + &PSI)); return false; } bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) { - Index.reset(); return false; } Index: lib/Bitcode/Writer/BitcodeWriterPass.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -21,7 +21,7 @@ PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { const ModuleSummaryIndex *Index = - EmitSummaryIndex ? &(AM.getResult(M)) + EmitSummaryIndex ? AM.getResult(M).get() : nullptr; WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash); return PreservedAnalyses::all(); Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -1055,7 +1055,7 @@ ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, ImportLists, ExportLists); - std::set ExportedGUIDs; + DenseSet ExportedGUIDs; for (auto &Res : GlobalResolutions) { // First check if the symbol was flagged as having external references. if (Res.second.Partition != GlobalResolution::External) Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -477,8 +477,9 @@ { raw_svector_ostream OS(OutputBuffer); ProfileSummaryInfo PSI(TheModule); - auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI); - WriteBitcodeToFile(&TheModule, OS, true, &Index); + std::unique_ptr Index( + buildModuleSummaryIndex(TheModule, nullptr, &PSI)); + WriteBitcodeToFile(&TheModule, OS, true, Index.get()); } return make_unique(std::move(OutputBuffer)); } Index: lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- lib/Transforms/IPO/FunctionImport.cpp +++ lib/Transforms/IPO/FunctionImport.cpp @@ -445,7 +445,7 @@ for (auto &S : Entry.second.SummaryList) if (S->isLive()) { DEBUG(dbgs() << "Live root: " << Entry.first << "\n"); - Worklist.push_back(ValueInfo(&Entry)); + Worklist.push_back(Index.getValueInfo(Entry.first)); ++LiveSymbols; break; } Index: lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp =================================================================== --- lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -352,13 +352,14 @@ // FIXME: Try to re-use BSI and PFI from the original module here. ProfileSummaryInfo PSI(M); - ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI); + std::unique_ptr Index( + buildModuleSummaryIndex(M, nullptr, &PSI)); // Mark the merged module as requiring full LTO. We still want an index for // it though, so that it can participate in summary-based dead stripping. MergedM->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); - ModuleSummaryIndex MergedMIndex = - buildModuleSummaryIndex(*MergedM, nullptr, &PSI); + std::unique_ptr MergedMIndex( + buildModuleSummaryIndex(*MergedM, nullptr, &PSI)); SmallVector Buffer; @@ -367,10 +368,10 @@ // be used in the backends, and use that in the minimized bitcode // produced for the full link. ModuleHash ModHash = {{0}}; - W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, + W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, Index.get(), /*GenerateHash=*/true, &ModHash); W.writeModule(MergedM.get(), /*ShouldPreserveUseListOrder=*/false, - &MergedMIndex); + MergedMIndex.get()); W.writeSymtab(); W.writeStrtab(); OS << Buffer; @@ -382,10 +383,10 @@ Buffer.clear(); BitcodeWriter W2(Buffer); StripDebugInfo(M); - W2.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, + W2.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, Index.get(), /*GenerateHash=*/false, &ModHash); W2.writeModule(MergedM.get(), /*ShouldPreserveUseListOrder=*/false, - &MergedMIndex); + MergedMIndex.get()); W2.writeSymtab(); W2.writeStrtab(); *ThinLinkOS << Buffer; @@ -485,6 +486,6 @@ [&FAM](Function &F) -> AAResults & { return FAM.getResult(F); }, - M, &AM.getResult(M)); + M, AM.getResult(M).get()); return PreservedAnalyses::all(); } Index: test/Bitcode/thinlto-alias.ll =================================================================== --- test/Bitcode/thinlto-alias.ll +++ test/Bitcode/thinlto-alias.ll @@ -26,8 +26,8 @@ ; COMBINED-NEXT: ; COMBINED-NEXT: -; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll +++ test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll @@ -48,10 +48,10 @@ ; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-type-tests.ll =================================================================== --- test/Bitcode/thinlto-type-tests.ll +++ test/Bitcode/thinlto-type-tests.ll @@ -4,8 +4,8 @@ ; RUN: llvm-bcanalyzer -dump %t2.thinlto.bc | FileCheck --check-prefix=COMBINED %s ; COMBINED: -; COMBINED: ; COMBINED: +; COMBINED: ; CHECK: define i1 @f() { Index: test/Bitcode/thinlto-type-vcalls.ll =================================================================== --- test/Bitcode/thinlto-type-vcalls.ll +++ test/Bitcode/thinlto-type-vcalls.ll @@ -6,17 +6,17 @@ target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" -; COMBINED: -; COMBINED-NEXT: +; COMBINED: ; COMBINED-NEXT: ; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED-NEXT: -; COMBINED-NEXT: ; COMBINED-NEXT: