diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h --- a/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -92,8 +92,9 @@ void addRecord(const sampleprof::FunctionSamples &FS, bool isCallsiteSample = false); - std::unique_ptr - computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles); + std::unique_ptr computeSummaryForProfiles( + const sampleprof::SampleProfileMap &Profiles, + bool ProfileIsCS = sampleprof::FunctionSamples::ProfileIsCS); std::unique_ptr getSummary(); }; diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h --- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h +++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h @@ -142,6 +142,9 @@ ContextTrieNode &getRootContext(); void promoteMergeContextSamplesTree(const Instruction &Inst, StringRef CalleeName); + + // Create a merged conext-less profile map. + void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles); // Dump the internal context profile trie. void dump(); @@ -158,7 +161,6 @@ promoteMergeContextSamplesTree(ContextTrieNode &FromNode, ContextTrieNode &ToNodeParent, uint32_t ContextFramesToRemove); - // Map from function name to context profiles (excluding base profile) StringMap FuncToCtxtProfiles; diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp --- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -185,7 +185,7 @@ std::unique_ptr SampleProfileSummaryBuilder::computeSummaryForProfiles( - const SampleProfileMap &Profiles) { + const SampleProfileMap &Profiles, bool ProfileIsCS) { assert(NumFunctions == 0 && "This can only be called on an empty summary builder"); sampleprof::SampleProfileMap ContextLessProfiles; @@ -196,8 +196,8 @@ // more function profiles each with lower counts, which in turn leads to lower // hot thresholds. To compensate for that, by default we merge context // profiles before computing profile summary. - if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && - !UseContextLessSummary.getNumOccurrences())) { + if (UseContextLessSummary || + (ProfileIsCS && !UseContextLessSummary.getNumOccurrences())) { for (const auto &I : Profiles) { ContextLessProfiles[I.second.getName()].merge(I.second); } diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp --- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp +++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp @@ -595,4 +595,24 @@ return *ToNode; } + +void SampleContextTracker::createContextLessProfileMap( + SampleProfileMap &ContextLessProfiles) { + std::queue NodeQueue; + NodeQueue.push(&RootContext); + + while (!NodeQueue.empty()) { + ContextTrieNode *Node = NodeQueue.front(); + FunctionSamples *FProfile = Node->getFunctionSamples(); + NodeQueue.pop(); + + if (FProfile) { + // Profile's context can be empty, use ContextNode's func name. + ContextLessProfiles[Node->getFuncName()].merge(*FProfile); + } + + for (auto &It : Node->getAllChildContext()) + NodeQueue.push(&It.second); + } +} } // namespace llvm diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h --- a/llvm/tools/llvm-profgen/ProfileGenerator.h +++ b/llvm/tools/llvm-profgen/ProfileGenerator.h @@ -32,6 +32,7 @@ class ProfileGeneratorBase { public: + ProfileGeneratorBase(ProfiledBinary *Binary) : Binary(Binary){}; ProfileGeneratorBase(ProfiledBinary *Binary, const ContextSampleCounterMap *Counters) : Binary(Binary), SampleCounters(Counters){}; @@ -44,7 +45,7 @@ create(ProfiledBinary *Binary, const ContextSampleCounterMap *Counters, bool profileIsCS); static std::unique_ptr - create(ProfiledBinary *Binary, const SampleProfileMap &&ProfileMap, + create(ProfiledBinary *Binary, SampleProfileMap &ProfileMap, bool profileIsCS); virtual void generateProfile() = 0; void write(); @@ -109,7 +110,8 @@ StringRef getCalleeNameForOffset(uint64_t TargetOffset); - void computeSummaryAndThreshold(); + void computeSummaryAndThreshold(SampleProfileMap &ProfileMap, + bool ProfileIsCS = false); void calculateAndShowDensity(const SampleProfileMap &Profiles); @@ -120,6 +122,8 @@ void collectProfiledFunctions(); + bool inputIsSampleProfile() { return SampleCounters == nullptr; } + // Thresholds from profile summary to answer isHotCount/isColdCount queries. uint64_t HotCountThreshold; @@ -173,8 +177,16 @@ CSProfileGenerator(ProfiledBinary *Binary, const ContextSampleCounterMap *Counters) : ProfileGeneratorBase(Binary, Counters){}; - CSProfileGenerator(ProfiledBinary *Binary, const SampleProfileMap &&Profiles) - : ProfileGeneratorBase(Binary, std::move(Profiles)){}; + CSProfileGenerator(ProfiledBinary *Binary, SampleProfileMap &Profiles) + : ProfileGeneratorBase(Binary), ContextTracker(Profiles, nullptr) { + // This is for the case the input is a llvm sample profile. + std::unordered_set ProfiledFunctions; + for (const auto &FS : Profiles) { + if (auto *Func = Binary->getBinaryFunction(FS.first.getName())) + ProfiledFunctions.insert(Func); + } + Binary->setProfiledFunctions(ProfiledFunctions); + }; void generateProfile() override; // Trim the context stack at a given depth. @@ -345,7 +357,7 @@ SampleContextFrameVector &Context); void buildProfileMap(); - + void computeSummaryAndThreshold(); ContextTrieNode &getRootContext() { return ContextTracker.getRootContext(); }; // Underlying context table serves for sample profile writer. diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -128,14 +128,13 @@ } std::unique_ptr -ProfileGeneratorBase::create(ProfiledBinary *Binary, - const SampleProfileMap &&Profiles, +ProfileGeneratorBase::create(ProfiledBinary *Binary, SampleProfileMap &Profiles, bool ProfileIsCS) { std::unique_ptr Generator; if (ProfileIsCS) { if (Binary->useFSDiscriminator()) exitWithError("FS discriminator is not supported in CS profile."); - Generator.reset(new CSProfileGenerator(Binary, std::move(Profiles))); + Generator.reset(new CSProfileGenerator(Binary, Profiles)); } else { Generator.reset(new ProfileGenerator(Binary, std::move(Profiles))); } @@ -431,15 +430,8 @@ ProfiledFunctions.insert(FRange->Func); } } - } else { - // This is for the case the input is a llvm sample profile. - for (const auto &FS : ProfileMap) { - if (auto *Func = Binary->getBinaryFunction(FS.first.getName())) - ProfiledFunctions.insert(Func); - } + Binary->setProfiledFunctions(ProfiledFunctions); } - - Binary->setProfiledFunctions(ProfiledFunctions); } FunctionSamples & @@ -471,7 +463,7 @@ } void ProfileGenerator::postProcessProfiles() { - computeSummaryAndThreshold(); + computeSummaryAndThreshold(ProfileMap); trimColdProfiles(ProfileMap, ColdCountThreshold); calculateAndShowDensity(ProfileMap); } @@ -944,7 +936,8 @@ auto Ret = ProfileMap.emplace(FContext, *FProfile); FunctionSamples &NewProfile = Ret.first->second; NewProfile.setContext(FContext); - delete (FProfile); + if (!inputIsSampleProfile()) + delete (FProfile); Node.setFunctionSamples(nullptr); Context.pop_back(); } @@ -964,12 +957,12 @@ } void CSProfileGenerator::postProcessProfiles() { - buildProfileMap(); - // Compute hot/cold threshold based on profile. This will be used for cold // context profile merging/trimming. computeSummaryAndThreshold(); + buildProfileMap(); + // Run global pre-inliner to adjust/merge context profile based on estimated // inline decisions. if (EnableCSPreInliner) { @@ -1001,15 +994,22 @@ } } -void ProfileGeneratorBase::computeSummaryAndThreshold() { +void ProfileGeneratorBase::computeSummaryAndThreshold( + SampleProfileMap &Profiles, bool ProfileIsCS) { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); - Summary = Builder.computeSummaryForProfiles(ProfileMap); + Summary = Builder.computeSummaryForProfiles(Profiles, ProfileIsCS); HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold( (Summary->getDetailedSummary())); ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold( (Summary->getDetailedSummary())); } +void CSProfileGenerator::computeSummaryAndThreshold() { + SampleProfileMap ContextLessProfiles; + ContextTracker.createContextLessProfileMap(ContextLessProfiles); + ProfileGeneratorBase::computeSummaryAndThreshold(ContextLessProfiles, false); +} + void ProfileGeneratorBase::extractProbesFromRange( const RangeSample &RangeCounter, ProbeCounterMap &ProbeCounter, bool FindDisjointRanges) { diff --git a/llvm/tools/llvm-profgen/llvm-profgen.cpp b/llvm/tools/llvm-profgen/llvm-profgen.cpp --- a/llvm/tools/llvm-profgen/llvm-profgen.cpp +++ b/llvm/tools/llvm-profgen/llvm-profgen.cpp @@ -167,8 +167,7 @@ std::move(ReaderOrErr.get()); Reader->read(); std::unique_ptr Generator = - ProfileGeneratorBase::create(Binary.get(), - std::move(Reader->getProfiles()), + ProfileGeneratorBase::create(Binary.get(), Reader->getProfiles(), Reader->profileIsCS()); Generator->generateProfile(); Generator->write();