Index: include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- include/llvm/Bitcode/LLVMBitCodes.h +++ include/llvm/Bitcode/LLVMBitCodes.h @@ -256,6 +256,10 @@ // strings in strtab. // [n * name] FS_CFI_FUNCTION_DECLS = 18, + // PERMODULE_RELBF: [valueid, flags, instcount, numrefs, + // numrefs x valueid, + // n x (valueid, relblockfreq)] + FS_PERMODULE_RELBF = 19, }; enum MetadataCodes { Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -55,13 +55,19 @@ Critical = 4 }; HotnessType Hotness = HotnessType::Unknown; + uint64_t RelBlockFreq = 0; CalleeInfo() = default; - explicit CalleeInfo(HotnessType Hotness) : Hotness(Hotness) {} + explicit CalleeInfo(HotnessType Hotness, uint64_t RelBF) + : Hotness(Hotness), RelBlockFreq(RelBF) {} void updateHotness(const HotnessType OtherHotness) { Hotness = std::max(Hotness, OtherHotness); } + + // When there are multiple edges between the same (caller, callee) pair, the + // relative block frequencies are summed up. + void updateRelBlockFreq(uint64_t RBF) { RelBlockFreq += RBF; } }; class GlobalValueSummary; Index: lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- lib/Analysis/ModuleSummaryAnalysis.cpp +++ lib/Analysis/ModuleSummaryAnalysis.cpp @@ -49,6 +49,7 @@ #include "llvm/Object/SymbolicFile.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" #include #include #include @@ -58,6 +59,9 @@ #define DEBUG_TYPE "module-summary-analysis" +cl::opt WriteRelBFToSummary( + "write-relbf-to-summary", cl::Hidden, cl::init(false), + cl::desc("Write relative block frequency to function summary ")); // Walk through the operands of a given User via worklist iteration and populate // the set of GlobalValue references encountered. Invoked either on an // Instruction or a GlobalVariable (which walks its initializer). @@ -276,6 +280,18 @@ CallGraphEdges[Index.getOrInsertValueInfo( cast(CalledValue))] .updateHotness(Hotness); + // Add the relative block frequency to CalleeInfo if there is no profile + // information and the -write-relbf-to-summary flag is set. + if (WriteRelBFToSummary && + Hotness == CalleeInfo::HotnessType::Unknown) { + auto BBFreq = BFI->getBlockFreq(CI->getParent()).getFrequency(); + // FIXME: This might need some scaling to prevent BBFreq values from + // being rounded down to 0. + BBFreq /= BFI->getEntryFreq(); + CallGraphEdges[Index.getOrInsertValueInfo( + cast(CalledValue))] + .updateRelBlockFreq(BBFreq); + } } else { // Skip inline assembly calls. if (CI && CI->isInlineAsm()) Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -743,7 +743,8 @@ std::vector makeRefList(ArrayRef Record); std::vector makeCallList(ArrayRef Record, bool IsOldProfileFormat, - bool HasProfile); + bool HasProfile, + bool HasRelBF); Error parseEntireSummary(unsigned ID); Error parseModuleStringTable(); @@ -5043,12 +5044,15 @@ return Ret; } -std::vector ModuleSummaryIndexBitcodeReader::makeCallList( - ArrayRef Record, bool IsOldProfileFormat, bool HasProfile) { +std::vector +ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef Record, + bool IsOldProfileFormat, + bool HasProfile, bool HasRelBF) { std::vector Ret; Ret.reserve(Record.size()); for (unsigned I = 0, E = Record.size(); I != E; ++I) { CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown; + uint64_t RelBF = 0; ValueInfo Callee = getValueInfoFromValueId(Record[I]).first; if (IsOldProfileFormat) { I += 1; // Skip old callsitecount field @@ -5056,7 +5060,9 @@ I += 1; // Skip old profilecount field } else if (HasProfile) Hotness = static_cast(Record[++I]); - Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}}); + else if (HasRelBF) + RelBF = Record[++I]; + Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo(Hotness, RelBF)}); } return Ret; } @@ -5135,7 +5141,11 @@ // FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, // n x (valueid, hotness)] + // FS_PERMODULE_RELBF: [valueid, flags, instcount, fflags, numrefs, + // numrefs x valueid, + // n x (valueid, relblockfreq)] case bitc::FS_PERMODULE: + case bitc::FS_PERMODULE_RELBF: case bitc::FS_PERMODULE_PROFILE: { unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; @@ -5161,9 +5171,10 @@ std::vector Refs = makeRefList( ArrayRef(Record).slice(RefListStartIndex, NumRefs)); bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE); + bool HasRelBF = (BitCode == bitc::FS_PERMODULE_RELBF); std::vector Calls = makeCallList( ArrayRef(Record).slice(CallGraphEdgeStartIndex), - IsOldProfileFormat, HasProfile); + IsOldProfileFormat, HasProfile, HasRelBF); auto FS = llvm::make_unique( Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), std::move(Calls), std::move(PendingTypeTests), @@ -5255,7 +5266,7 @@ bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE); std::vector Edges = makeCallList( ArrayRef(Record).slice(CallGraphEdgeStartIndex), - IsOldProfileFormat, HasProfile); + IsOldProfileFormat, HasProfile, false); ValueInfo VI = getValueInfoFromValueId(ValueID).first; auto FS = llvm::make_unique( Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -86,6 +86,7 @@ cl::desc("Number of metadatas above which we emit an index " "to enable lazy-loading")); +extern cl::opt WriteRelBFToSummary; namespace { /// These are manifest constants used by the bitcode writer. They do not need to @@ -3377,11 +3378,15 @@ NameVals.push_back(getValueId(ECI.first)); if (HasProfileData) NameVals.push_back(static_cast(ECI.second.Hotness)); + else if (WriteRelBFToSummary) + NameVals.push_back(ECI.second.RelBlockFreq); } unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); unsigned Code = - (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE); + (HasProfileData ? bitc::FS_PERMODULE_PROFILE + : (WriteRelBFToSummary ? bitc::FS_PERMODULE_RELBF + : bitc::FS_PERMODULE)); // Emit the finished record. Stream.EmitRecord(Code, NameVals, FSAbbrev); @@ -3447,31 +3452,34 @@ ArrayRef{GVI.second, GVI.first}); } - // Abbrev for FS_PERMODULE. + // Abbrev for FS_PERMODULE_PROFILE. auto Abbv = std::make_shared(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid) + // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - // Abbrev for FS_PERMODULE_PROFILE. + // Abbrev for FS_PERMODULE or FS_PERMODULE_RELBF. Abbv = std::make_shared(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); + if (WriteRelBFToSummary) + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF)); + else + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, hotness) + // numrefs x valueid, n x (valueid [, rel_block_freq]) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. Abbv = std::make_shared(); Index: test/Bitcode/thinlto-function-summary-callgraph-relbf.ll =================================================================== --- /dev/null +++ test/Bitcode/thinlto-function-summary-callgraph-relbf.ll @@ -0,0 +1,35 @@ +; Test to check the callgraph in summary +; RUN: opt -write-relbf-to-summary -module-summary %s -o %t.o +; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s + + +; CHECK: +; CHECK: