Index: include/llvm/ProfileData/SampleProf.h =================================================================== --- include/llvm/ProfileData/SampleProf.h +++ include/llvm/ProfileData/SampleProf.h @@ -293,6 +293,9 @@ /// with the maximum total sample count. const FunctionSamples *findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName) const { + std::string CalleeGUID; + CalleeName = getRepInFormat(CalleeName, Format, CalleeGUID); + auto iter = CallsiteSamples.find(Loc); if (iter == CallsiteSamples.end()) return nullptr; @@ -377,23 +380,23 @@ /// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID /// to \p S. void findInlinedFunctions(DenseSet &S, const Module *M, - uint64_t Threshold, bool isCompact) const { + uint64_t Threshold) const { if (TotalSamples <= Threshold) return; - S.insert(Function::getGUID(Name)); + S.insert(getGUID(Name)); // Import hot CallTargets, which may not be available in IR because full // profile annotation cannot be done until backend compilation in ThinLTO. for (const auto &BS : BodySamples) for (const auto &TS : BS.second.getCallTargets()) if (TS.getValue() > Threshold) { - Function *Callee = M->getFunction(TS.getKey()); + const Function *Callee = + M->getFunction(getNameInModule(TS.getKey(), M)); if (!Callee || !Callee->getSubprogram()) - S.insert(isCompact ? std::stol(TS.getKey().data()) - : Function::getGUID(TS.getKey())); + S.insert(getGUID(TS.getKey())); } for (const auto &CS : CallsiteSamples) for (const auto &NameFS : CS.second) - NameFS.second.findInlinedFunctions(S, M, Threshold, isCompact); + NameFS.second.findInlinedFunctions(S, M, Threshold); } /// Set the name of the function. @@ -402,6 +405,29 @@ /// Return the function name. const StringRef &getName() const { return Name; } + /// Return the original function name if it exists in Module \p M. + StringRef getFuncNameInModule(const Module *M) const { + return getNameInModule(Name, M); + } + + /// Translate \p Name into its original name in Module. + /// When the Format is not SPF_Compact_Binary, \p Name needs no translation. + /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples + /// is actually GUID of the original function name. getNameInModule will + /// translate \p Name in current FunctionSamples into its original name. + /// If the original name doesn't exist in \p M, return empty StringRef. + StringRef getNameInModule(StringRef Name, const Module *M) const { + if (Format != SPF_Compact_Binary) + return Name; + // Expect CurrentModule to be initialized by GUIDToFuncNameMapper. + if (M != CurrentModule) + llvm_unreachable("Input Module should be the same as CurrentModule"); + auto iter = GUIDToFuncNameMap.find(std::stoul(Name.data())); + if (iter == GUIDToFuncNameMap.end()) + return StringRef(); + return iter->second; + } + /// Returns the line offset to the start line of the subprogram. /// We assume that a single function will not exceed 65535 LOC. static unsigned getOffset(const DILocation *DIL); @@ -417,6 +443,54 @@ /// \returns the FunctionSamples pointer to the inlined instance. const FunctionSamples *findFunctionSamples(const DILocation *DIL) const; + static SampleProfileFormat Format; + /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for + /// all the function symbols defined or declared in CurrentModule. + static DenseMap GUIDToFuncNameMap; + static Module *CurrentModule; + + class GUIDToFuncNameMapper { + public: + GUIDToFuncNameMapper(Module &M) { + if (Format != SPF_Compact_Binary) + return; + + for (const auto &F : M) { + StringRef OrigName = F.getName(); + GUIDToFuncNameMap.insert({Function::getGUID(OrigName), OrigName}); + /// Local to global var promotion used by optimization like thinlto + /// will rename the var and add suffix like ".llvm.xxx" to the + /// original local name. In sample profile, the suffixes of function + /// names are all stripped. Since it is possible that the mapper is + /// built in post-thin-link phase and var promotion has been done, + /// we need to add the substring of function name without the suffix + /// into the GUIDToFuncNameMap. + auto pos = OrigName.find('.'); + if (pos != StringRef::npos) { + StringRef NewName = OrigName.substr(0, pos); + GUIDToFuncNameMap.insert({Function::getGUID(NewName), NewName}); + } + } + CurrentModule = &M; + } + + ~GUIDToFuncNameMapper() { + if (Format != SPF_Compact_Binary) + return; + + GUIDToFuncNameMap.clear(); + CurrentModule = nullptr; + } + }; + + // Assume the input \p Name is a name coming from FunctionSamples itself. + // If the format is SPF_Compact_Binary, the name is already a GUID and we + // don't want to return the GUID of GUID. + static uint64_t getGUID(const StringRef &Name) { + return (Format == SPF_Compact_Binary) ? std::stoul(Name.data()) + : Function::getGUID(Name); + } + private: /// Mangled name of the function. StringRef Name; Index: lib/ProfileData/SampleProf.cpp =================================================================== --- lib/ProfileData/SampleProf.cpp +++ lib/ProfileData/SampleProf.cpp @@ -26,6 +26,14 @@ using namespace llvm; using namespace sampleprof; +namespace llvm { +namespace sampleprof { +SampleProfileFormat FunctionSamples::Format; +DenseMap FunctionSamples::GUIDToFuncNameMap; +Module *FunctionSamples::CurrentModule; +} // namespace sampleprof +} // namespace llvm + namespace { // FIXME: This class is only here to support the transition to llvm::Error. It Index: lib/ProfileData/SampleProfReader.cpp =================================================================== --- lib/ProfileData/SampleProfReader.cpp +++ lib/ProfileData/SampleProfReader.cpp @@ -880,6 +880,7 @@ else return sampleprof_error::unrecognized_format; + FunctionSamples::Format = Reader->getFormat(); if (std::error_code EC = Reader->readHeader()) return EC; Index: lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- lib/Transforms/IPO/SampleProfile.cpp +++ lib/Transforms/IPO/SampleProfile.cpp @@ -643,8 +643,6 @@ if (FS == nullptr) return nullptr; - std::string CalleeGUID; - CalleeName = getRepInFormat(CalleeName, Reader->getFormat(), CalleeGUID); return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL), DIL->getBaseDiscriminator()), CalleeName); @@ -685,7 +683,10 @@ } llvm::sort(R.begin(), R.end(), [](const FunctionSamples *L, const FunctionSamples *R) { - return L->getEntrySamples() > R->getEntrySamples(); + if (L->getEntrySamples() != R->getEntrySamples()) + return L->getEntrySamples() > R->getEntrySamples(); + return FunctionSamples::getGUID(L->getName()) < + FunctionSamples::getGUID(R->getName()); }); } return R; @@ -760,7 +761,6 @@ Function &F, DenseSet &InlinedGUIDs) { DenseSet PromotedInsns; bool Changed = false; - bool isCompact = (Reader->getFormat() == SPF_Compact_Binary); while (true) { bool LocalChanged = false; SmallVector CIS; @@ -792,19 +792,16 @@ for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) { if (IsThinLTOPreLink) { FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), - PSI->getOrCompHotCountThreshold(), - isCompact); + PSI->getOrCompHotCountThreshold()); continue; } - auto CalleeFunctionName = FS->getName(); + auto CalleeFunctionName = FS->getFuncNameInModule(F.getParent()); // If it is a recursive call, we do not inline it as it could bloat // the code exponentially. There is way to better handle this, e.g. // clone the caller first, and inline the cloned caller if it is // recursive. As llvm does not inline recursive calls, we will // simply ignore it instead of handling it explicitly. - std::string FGUID; - auto Fname = getRepInFormat(F.getName(), Reader->getFormat(), FGUID); - if (CalleeFunctionName == Fname) + if (CalleeFunctionName == F.getName()) continue; const char *Reason = "Callee function not available"; @@ -834,8 +831,7 @@ LocalChanged = true; } else if (IsThinLTOPreLink) { findCalleeFunctionSamples(*I)->findInlinedFunctions( - InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold(), - isCompact); + InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); } } if (LocalChanged) { @@ -1177,7 +1173,7 @@ const SampleRecord::CallTargetMap &M) { SmallVector R; for (auto I = M.begin(); I != M.end(); ++I) - R.push_back({Function::getGUID(I->getKey()), I->getValue()}); + R.push_back({FunctionSamples::getGUID(I->getKey()), I->getValue()}); llvm::sort(R.begin(), R.end(), [](const InstrProfValueData &L, const InstrProfValueData &R) { if (L.Count == R.Count) @@ -1533,6 +1529,7 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, ProfileSummaryInfo *_PSI) { + FunctionSamples::GUIDToFuncNameMapper Mapper(M); if (!ProfileIsValid) return false; Index: test/Transforms/SampleProfile/Inputs/function_metadata.prof =================================================================== --- test/Transforms/SampleProfile/Inputs/function_metadata.prof +++ test/Transforms/SampleProfile/Inputs/function_metadata.prof @@ -1,13 +1,17 @@ -test:10000:0 +test:3200:0 + 1: 100 2: 100 - 3: 100 3: foo:1000 + 1: 800 3: bar:200 + 2: 190 4: baz:10 + 2: 10 4: foo1:1000 1: 1000 4: foo2:1000 1: 1000 foo3:1000 -test_liveness:10000:0 +test_liveness:1000:0 1: foo:1000 1: foo_available:1000 + 2: 1000 Index: test/Transforms/SampleProfile/function_metadata.ll =================================================================== --- test/Transforms/SampleProfile/function_metadata.ll +++ test/Transforms/SampleProfile/function_metadata.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -passes='thinlto-pre-link' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck %s +; RUN: opt < %s -passes='thinlto-pre-link' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s ; Tests whether the functions in the inline stack are added to the ; function_entry_count metadata. Index: test/Transforms/SampleProfile/indirect-call.ll =================================================================== --- test/Transforms/SampleProfile/indirect-call.ll +++ test/Transforms/SampleProfile/indirect-call.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.prof -S | FileCheck %s +; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.compact.afdo -S | FileCheck %s ; CHECK-LABEL: @test define void @test(void ()*) !dbg !3 {