Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -499,8 +499,9 @@ FunctionSummary::GVFlags( GlobalValue::LinkageTypes::AvailableExternallyLinkage, /*NotEligibleToImport=*/true, /*Live=*/true, /*IsLocal=*/false), - 0, FunctionSummary::FFlags{}, std::vector(), - std::move(Edges), std::vector(), + /*InsCount=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0, + std::vector(), std::move(Edges), + std::vector(), std::vector(), std::vector(), std::vector(), @@ -518,6 +519,11 @@ /// Function summary specific flags. FFlags FunFlags; + /// The synthesized entry count of the function. + /// This is only populated during ThinLink phase and remains unused while + /// generating per-module summaries. + uint64_t EntryCount = 0; + /// List of call edge pairs from this function. std::vector CallGraphEdgeList; @@ -525,14 +531,15 @@ public: FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags, - std::vector Refs, std::vector CGEdges, + uint64_t EntryCount, std::vector Refs, + std::vector CGEdges, std::vector TypeTests, std::vector TypeTestAssumeVCalls, std::vector TypeCheckedLoadVCalls, std::vector TypeTestAssumeConstVCalls, std::vector TypeCheckedLoadConstVCalls) : GlobalValueSummary(FunctionKind, Flags, std::move(Refs)), - InstCount(NumInsts), FunFlags(FunFlags), + InstCount(NumInsts), FunFlags(FunFlags), EntryCount(EntryCount), CallGraphEdgeList(std::move(CGEdges)) { if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() || !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() || @@ -555,6 +562,12 @@ /// Get the instruction count recorded for this function. unsigned instCount() const { return InstCount; } + /// Get the synthetic entry count for this function. + uint64_t entryCount() const { return EntryCount; } + + /// Set the synthetic entry count for this function. + void setEntryCount(uint64_t EC) { EntryCount = EC; } + /// Return the list of pairs. ArrayRef calls() const { return CallGraphEdgeList; } @@ -1140,6 +1153,7 @@ /// GraphTraits definition to build SCC for the index template <> struct GraphTraits { typedef ValueInfo NodeRef; + using EdgeRef = FunctionSummary::EdgeTy &; static NodeRef valueInfoFromEdge(FunctionSummary::EdgeTy &P) { return P.first; @@ -1148,6 +1162,8 @@ mapped_iterator::iterator, decltype(&valueInfoFromEdge)>; + using ChildEdgeIteratorType = std::vector::iterator; + static NodeRef getEntryNode(ValueInfo V) { return V; } static ChildIteratorType child_begin(NodeRef N) { @@ -1169,6 +1185,26 @@ cast(N.getSummaryList().front()->getBaseObject()); return ChildIteratorType(F->CallGraphEdgeList.end(), &valueInfoFromEdge); } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + if (!N.getSummaryList().size()) // handle external function + return FunctionSummary::ExternalNode.CallGraphEdgeList.begin(); + + FunctionSummary *F = + cast(N.getSummaryList().front()->getBaseObject()); + return F->CallGraphEdgeList.begin(); + } + + static ChildEdgeIteratorType child_edge_end(NodeRef N) { + if (!N.getSummaryList().size()) // handle external function + return FunctionSummary::ExternalNode.CallGraphEdgeList.end(); + + FunctionSummary *F = + cast(N.getSummaryList().front()->getBaseObject()); + return F->CallGraphEdgeList.end(); + } + + static NodeRef edge_dest(EdgeRef E) { return E.first; } }; template <> Index: include/llvm/IR/ModuleSummaryIndexYAML.h =================================================================== --- include/llvm/IR/ModuleSummaryIndexYAML.h +++ include/llvm/IR/ModuleSummaryIndexYAML.h @@ -224,7 +224,7 @@ GlobalValueSummary::GVFlags( static_cast(FSum.Linkage), FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal), - 0, FunctionSummary::FFlags{}, Refs, + /*NumInsts=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0, Refs, ArrayRef{}, std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls), std::move(FSum.TypeCheckedLoadVCalls), Index: include/llvm/Transforms/IPO/FunctionImport.h =================================================================== --- include/llvm/Transforms/IPO/FunctionImport.h +++ include/llvm/Transforms/IPO/FunctionImport.h @@ -180,6 +180,9 @@ /// it is an alias. Returns true if converted, false if replaced. bool convertToDeclaration(GlobalValue &GV); +/// Compute synthetic function entry counts. +void computeSyntheticCounts(ModuleSummaryIndex &Index); + /// Compute the set of summaries needed for a ThinLTO backend compilation of /// \p ModulePath. // Index: lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- lib/Analysis/ModuleSummaryAnalysis.cpp +++ lib/Analysis/ModuleSummaryAnalysis.cpp @@ -368,7 +368,7 @@ // Don't try to import functions with noinline attribute. F.getAttributes().hasFnAttribute(Attribute::NoInline)}; auto FuncSummary = llvm::make_unique( - Flags, NumInsts, FunFlags, RefEdges.takeVector(), + Flags, NumInsts, FunFlags, /*EntryCount=*/0, RefEdges.takeVector(), CallGraphEdges.takeVector(), TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(), TypeTestAssumeConstVCalls.takeVector(), @@ -476,14 +476,15 @@ if (Function *F = dyn_cast(GV)) { std::unique_ptr Summary = llvm::make_unique( - GVFlags, 0, + GVFlags, /*InsCount=*/0, FunctionSummary::FFlags{ F->hasFnAttribute(Attribute::ReadNone), F->hasFnAttribute(Attribute::ReadOnly), F->hasFnAttribute(Attribute::NoRecurse), F->returnDoesNotAlias(), /* NoInline = */ false}, - ArrayRef{}, ArrayRef{}, + /*EntryCount=*/0, ArrayRef{}, + ArrayRef{}, ArrayRef{}, ArrayRef{}, ArrayRef{}, Index: lib/Analysis/SyntheticCountsUtils.cpp =================================================================== --- lib/Analysis/SyntheticCountsUtils.cpp +++ lib/Analysis/SyntheticCountsUtils.cpp @@ -14,12 +14,12 @@ #include "llvm/Analysis/SyntheticCountsUtils.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SCCIterator.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/ModuleSummaryIndex.h" using namespace llvm; @@ -29,7 +29,7 @@ const SccTy &SCC, GetRelBBFreqTy GetRelBBFreq, GetCountTy GetCount, AddCountTy AddCount) { - SmallPtrSet SCCNodes; + DenseSet SCCNodes; SmallVector, 8> SCCEdges, NonSCCEdges; for (auto &Node : SCC) @@ -111,3 +111,4 @@ } template class llvm::SyntheticCountsUtils; +template class llvm::SyntheticCountsUtils; Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -7672,8 +7672,8 @@ return true; auto FS = llvm::make_unique( - GVFlags, InstCount, FFlags, std::move(Refs), std::move(Calls), - std::move(TypeIdInfo.TypeTests), + GVFlags, InstCount, FFlags, /*EntryCount=*/0, std::move(Refs), + std::move(Calls), std::move(TypeIdInfo.TypeTests), std::move(TypeIdInfo.TypeTestAssumeVCalls), std::move(TypeIdInfo.TypeCheckedLoadVCalls), std::move(TypeIdInfo.TypeTestAssumeConstVCalls), Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -5236,9 +5236,9 @@ } const uint64_t Version = Record[0]; const bool IsOldProfileFormat = Version == 1; - if (Version < 1 || Version > 4) + if (Version < 1 || Version > 5) return error("Invalid summary version " + Twine(Version) + - ", 1, 2, 3 or 4 expected"); + ". Version should be in the range [1-5]."); Record.clear(); // Keep around the last seen summary to be used when we see an optional @@ -5341,8 +5341,8 @@ ArrayRef(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile, HasRelBF); auto FS = llvm::make_unique( - Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), - std::move(Calls), std::move(PendingTypeTests), + Flags, InstCount, getDecodedFFlags(RawFunFlags), /*EntryCount=*/0, + std::move(Refs), std::move(Calls), std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), @@ -5413,13 +5413,18 @@ uint64_t RawFlags = Record[2]; unsigned InstCount = Record[3]; uint64_t RawFunFlags = 0; + uint64_t EntryCount = 0; unsigned NumRefs = Record[4]; int RefListStartIndex = 5; if (Version >= 4) { RawFunFlags = Record[4]; - NumRefs = Record[5]; RefListStartIndex = 6; + if (Version >= 5) { + EntryCount = Record[5]; + RefListStartIndex = 7; + } + NumRefs = Record[RefListStartIndex - 1]; } auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); @@ -5434,8 +5439,8 @@ IsOldProfileFormat, HasProfile, false); ValueInfo VI = getValueInfoFromValueId(ValueID).first; auto FS = llvm::make_unique( - Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), - std::move(Edges), std::move(PendingTypeTests), + Flags, InstCount, getDecodedFFlags(RawFunFlags), EntryCount, + std::move(Refs), std::move(Edges), std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3596,7 +3596,7 @@ // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 4; +static const uint64_t INDEX_VERSION = 5; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. @@ -3740,6 +3740,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3851,6 +3852,8 @@ NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); NameVals.push_back(getEncodedFFlags(FS->fflags())); + NameVals.push_back(FS->entryCount()); + // Fill in below NameVals.push_back(0); @@ -3862,7 +3865,7 @@ NameVals.push_back(*RefValueId); Count++; } - NameVals[5] = Count; + NameVals[6] = Count; bool HasProfileData = false; for (auto &EI : FS->calls()) { Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -1157,6 +1157,8 @@ if (!ModuleToDefinedGVSummaries.count(Mod.first)) ModuleToDefinedGVSummaries.try_emplace(Mod.first); + computeSyntheticCounts(ThinLTO.CombinedIndex); + StringMap ImportLists( ThinLTO.ModuleMap.size()); StringMap ExportLists( Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -199,7 +199,8 @@ static void crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index, StringMap &ModuleMap, - const FunctionImporter::ImportMapTy &ImportList) { + const FunctionImporter::ImportMapTy &ImportList, + const GVSummaryMapTy &DefinedGlobals) { auto Loader = [&](StringRef Identifier) { return loadModuleFromBuffer(ModuleMap[Identifier], TheModule.getContext(), /*Lazy=*/true, /*IsImporting*/ true); @@ -475,7 +476,8 @@ saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc"); if (!SingleModule) { - crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); + crossImportIntoModule(TheModule, Index, ModuleMap, ImportList, + DefinedGlobals); // Save temps: after cross-module import. saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc"); @@ -714,8 +716,11 @@ ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); auto &ImportList = ImportLists[TheModule.getModuleIdentifier()]; + auto &DefinedGlobals = + ModuleToDefinedGVSummaries[TheModule.getModuleIdentifier()]; - crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); + crossImportIntoModule(TheModule, Index, ModuleMap, ImportList, + DefinedGlobals); } /** Index: lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- lib/Transforms/IPO/FunctionImport.cpp +++ lib/Transforms/IPO/FunctionImport.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" +#include "llvm/Analysis/SyntheticCountsUtils.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/Constants.h" @@ -145,6 +146,8 @@ ImportAllIndex("import-all-index", cl::desc("Import all external functions in index.")); +extern cl::opt InitialSyntheticCount; + // Load lazily a module from \p FileName in \p Context. static std::unique_ptr loadFile(const std::string &FileName, LLVMContext &Context) { @@ -824,6 +827,57 @@ NumLiveSymbols += LiveSymbols; } +static void initializeCounts(ModuleSummaryIndex &Index) { + auto Root = Index.calculateCallGraphRoot(); + // Root is a fake node. All its successors are the actual roots of the + // callgraph. + // FIXME: This initializes the entry counts of only the root nodes. This makes + // sense when compiling a binary with ThinLTO, but for libraries any of the + // non-root nodes could be called from outside. + for (auto &C : Root.calls()) { + auto &V = C.first; + for (auto &GVS : V.getSummaryList()) { + auto S = GVS.get()->getBaseObject(); + auto *F = cast(S); + F->setEntryCount(InitialSyntheticCount); + } + } +} + +void llvm::computeSyntheticCounts(ModuleSummaryIndex &Index) { + using Scaled64 = ScaledNumber; + initializeCounts(Index); + auto GetCallSiteRelFreq = [](FunctionSummary::EdgeTy &Edge) { + // FIXME: Handle fake/invalid edges? + return Scaled64(Edge.second.RelBlockFreq, -CalleeInfo::ScaleShift); + }; + auto GetEntryCount = [](ValueInfo V) { + if (V.getSummaryList().size()) { + auto S = V.getSummaryList().front().get()->getBaseObject(); + auto *F = cast(S); + return F->entryCount(); + } else { + return UINT64_C(0); + } + }; + auto AddToEntryCount = [](ValueInfo V, uint64_t New) { + if (!V.getSummaryList().size()) + return; + for (auto &GVS : V.getSummaryList()) { + auto S = GVS.get()->getBaseObject(); + auto *F = cast(S); + F->setEntryCount(SaturatingAdd(F->entryCount(), New)); + } + }; + + // After initializing the counts in initializeCounts above, the counts have to + // be propagated across the combined callgraph. + // SyntheticCountsUtils::propagate takes care of this propagation on any + // callgraph that specialized GraphTraits. + SyntheticCountsUtils::propagate( + &Index, GetCallSiteRelFreq, GetEntryCount, AddToEntryCount); +} + /// Compute the set of summaries needed for a ThinLTO backend compilation of /// \p ModulePath. void llvm::gatherImportedSummariesForModule( Index: lib/Transforms/IPO/SyntheticCountsPropagation.cpp =================================================================== --- lib/Transforms/IPO/SyntheticCountsPropagation.cpp +++ lib/Transforms/IPO/SyntheticCountsPropagation.cpp @@ -46,7 +46,7 @@ #define DEBUG_TYPE "synthetic-counts-propagation" /// Initial synthetic count assigned to functions. -static cl::opt +cl::opt InitialSyntheticCount("initial-synthetic-count", cl::Hidden, cl::init(10), cl::ZeroOrMore, cl::desc("Initial value of synthetic entry count.")); Index: lib/Transforms/Utils/FunctionImportUtils.cpp =================================================================== --- lib/Transforms/Utils/FunctionImportUtils.cpp +++ lib/Transforms/Utils/FunctionImportUtils.cpp @@ -16,6 +16,10 @@ #include "llvm/IR/InstIterator.h" using namespace llvm; +cl::opt EnableImportEntryCount( + "enable-import-entry-count", cl::init(false), cl::Hidden, + cl::desc("Attach 'synthetic_entry_count' metadata after importing")); + /// Checks if we should import SGV as a definition, otherwise import as a /// declaration. bool FunctionImportGlobalProcessing::doImportAsDefinition( @@ -202,10 +206,26 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) { - // Check the summaries to see if the symbol gets resolved to a known local - // definition. if (GV.hasName()) { ValueInfo VI = ImportIndex.getValueInfo(GV.getGUID()); + // Set synthetic function entry counts. + if (VI && EnableImportEntryCount) { + if (Function *F = dyn_cast(&GV)) { + if (!F->isDeclaration()) { + for (auto &S : VI.getSummaryList()) { + FunctionSummary *FS = dyn_cast(S->getBaseObject()); + if (FS->modulePath() == M.getModuleIdentifier()) { + F->setEntryCount(Function::ProfileCount(FS->entryCount(), + Function::PCT_Synthetic)); + break; + } + + } + } + } + } + // Check the summaries to see if the symbol gets resolved to a known local + // definition. if (VI && VI.isDSOLocal()) { GV.setDSOLocal(true); if (GV.hasDLLImportStorageClass()) Index: test/Bitcode/summary_version.ll =================================================================== --- test/Bitcode/summary_version.ll +++ test/Bitcode/summary_version.ll @@ -2,7 +2,7 @@ ; RUN: opt -module-summary %s -o - | llvm-bcanalyzer -dump | FileCheck %s ; CHECK: +; CHECK: Index: test/Bitcode/thinlto-alias.ll =================================================================== --- test/Bitcode/thinlto-alias.ll +++ test/Bitcode/thinlto-alias.ll @@ -33,7 +33,7 @@ ; COMBINED-NEXT: ; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll +++ test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll @@ -71,7 +71,7 @@ ; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll +++ test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll @@ -58,7 +58,7 @@ ; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph.ll +++ test/Bitcode/thinlto-function-summary-callgraph.ll @@ -33,7 +33,7 @@ ; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/ThinLTO/X86/Inputs/function_entry_count.ll =================================================================== --- /dev/null +++ test/ThinLTO/X86/Inputs/function_entry_count.ll @@ -0,0 +1,9 @@ +target triple = "x86_64-unknown-linux-gnu" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" + +declare void @h(); + +define void @g() { + call void @h(); + ret void +} Index: test/ThinLTO/X86/function_entry_count.ll =================================================================== --- /dev/null +++ test/ThinLTO/X86/function_entry_count.ll @@ -0,0 +1,40 @@ +; RUN: opt -thinlto-bc %s -write-relbf-to-summary -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc +; RUN: opt -thinlto-bc %p/Inputs/function_entry_count.ll -write-relbf-to-summary -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc + +; First perform the thin link on the normal bitcode file. +; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps -enable-import-entry-count \ +; RUN: -r=%t1.bc,g, \ +; RUN: -r=%t1.bc,f,px \ +; RUN: -r=%t1.bc,h,px \ +; RUN: -r=%t2.bc,h, \ +; RUN: -r=%t2.bc,g,px +; RUN: llvm-dis -o - %t.o.1.3.import.bc | FileCheck %s + +; CHECK: define void @h() !prof ![[PROF2:[0-9]+]] +; CHECK: define void @f(i32 %n) !prof ![[PROF1:[0-9]+]] +; CHECK: define available_externally void @g() !prof ![[PROF2]] +; CHECK-DAG: ![[PROF1]] = !{!"synthetic_function_entry_count", i64 10} +; CHECK-DAG: ![[PROF2]] = !{!"synthetic_function_entry_count", i64 198} + +target triple = "x86_64-unknown-linux-gnu" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" + +declare void @g(); + +define void @h() { + ret void +} + +define void @f(i32 %n) { +entry: + %cmp = icmp slt i32 %n, 1 + br i1 %cmp, label %exit, label %loop +loop: + %n1 = phi i32 [%n, %entry], [%n2, %loop] + call void @g() + %n2 = sub i32 %n1, 1 + %cmp2 = icmp slt i32 %n, 1 + br i1 %cmp2, label %exit, label %loop +exit: + ret void +}