diff --git a/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test b/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test --- a/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test +++ b/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test @@ -1,4 +1,19 @@ ; RUN: llvm-profgen --perfscript=%S/Inputs/inline-cs-pseudoprobe.perfscript --binary=%S/Inputs/inline-cs-pseudoprobe.perfbin --output=%t --show-unwinder-output | FileCheck %s --check-prefix=CHECK-UNWINDER +; RUN: FileCheck %s --input-file %t + +; CHECK:[main:2 @ foo]:74:0 +; CHECK: 2: 15 +; CHECK: 3: 15 +; CHECK: 4: 14 +; CHECK: 5: 1 +; CHECK: 6: 15 +; CHECK: 8: 14 bar:14 +; CHECK:[main:2 @ foo:8 @ bar]:28:14 +; CHECK: 1: 14 +; CHECK: 2: 0 +; CHECK: 3: 0 +; CHECK: 4: 14 + ; CHECK-UNWINDER: Binary(inline-cs-pseudoprobe.perfbin)'s Range Counter: ; CHECK-UNWINDER: (800, 858): 1 diff --git a/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test b/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test --- a/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test +++ b/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test @@ -1,5 +1,17 @@ ; RUN: llvm-profgen --perfscript=%S/Inputs/noinline-cs-pseudoprobe.perfscript --binary=%S/Inputs/noinline-cs-pseudoprobe.perfbin --output=%t --show-unwinder-output | FileCheck %s --check-prefix=CHECK-UNWINDER - +; RUN: FileCheck %s --input-file %t + +; CHECK:[main:2 @ foo]:75:0 +; CHECK: 2: 15 +; CHECK: 3: 15 +; CHECK: 4: 15 +; CHECK: 6: 15 +; CHECK: 8: 15 bar:15 +; CHECK:[main:2 @ foo:8 @ bar]:30:15 +; CHECK: 1: 15 +; CHECK: 2: 0 +; CHECK: 3: 0 +; CHECK: 4: 15 ; CHECK-UNWINDER: Binary(noinline-cs-pseudoprobe.perfbin)'s Range Counter: ; CHECK-UNWINDER: main:2 diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h --- a/llvm/tools/llvm-profgen/ProfileGenerator.h +++ b/llvm/tools/llvm-profgen/ProfileGenerator.h @@ -25,7 +25,7 @@ ProfileGenerator(){}; virtual ~ProfileGenerator() = default; static std::unique_ptr - create(const BinarySampleCounterMap &SampleCounters, + create(const BinarySampleCounterMap &BinarySampleCounters, enum PerfScriptType SampleType); virtual void generateProfile() = 0; @@ -50,7 +50,6 @@ */ void findDisjointRanges(RangeSample &DisjointRanges, const RangeSample &Ranges); - // Used by SampleProfileWriter StringMap ProfileMap; }; @@ -90,14 +89,16 @@ populateInferredFunctionSamples(); } +protected: + // Lookup or create FunctionSamples for the context + FunctionSamples &getFunctionProfileForContext(StringRef ContextId); + private: // Helper function for updating body sample for a leaf location in // FunctionProfile void updateBodySamplesforFunctionProfile(FunctionSamples &FunctionProfile, const FrameLocation &LeafLoc, uint64_t Count); - // Lookup or create FunctionSamples for the context - FunctionSamples &getFunctionProfileForContext(StringRef ContextId); void populateFunctionBodySamples(FunctionSamples &FunctionProfile, const RangeSample &RangeCounters, ProfiledBinary *Binary); @@ -108,14 +109,37 @@ void populateInferredFunctionSamples(); }; +using ProbeCounterMap = std::unordered_map; + class PseudoProbeProfileGenerator : public CSProfileGenerator { public: PseudoProbeProfileGenerator(const BinarySampleCounterMap &Counters) : CSProfileGenerator(Counters) {} - void generateProfile() override { - // TODO - } + void generateProfile() override; + +private: + // Go throught each address from range to extract the top frame probe by + // looking up in the Address2ProbeMap + void extractProbesFromRange(const RangeSample &RangeCounter, + ProbeCounterMap &ProbeCounter, + ProfiledBinary *Binary); + // Fill in function body samples from probes + void populateBodySamplesWithProbes(const RangeSample &RangeCounter, + StringRef PrefixContextId, + ProfiledBinary *Binary); + // Fill in boudary samples for a call probe + void populateBoundarySamplesWithProbes(const BranchSample &BranchCounter, + StringRef PrefixContextId, + ProfiledBinary *Binary); + // Helper function to get FunctionSamples for the leaf inlined context + FunctionSamples & + getFunctionProfileForLeaf(StringRef PrefixContextId, + SmallVector &LeafInlinedContext); + // Helper function to get FunctionSamples for the leaf probe + FunctionSamples &getFunctionProfileForLeafProbe(StringRef PrefixContextId, + const PseudoProbe *LeafProbe, + ProfiledBinary *Binary); }; } // end namespace sampleprof diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -316,5 +316,182 @@ } } +// Helper function to extract context prefix +// Remind that the string in ContextStrStack is in callee-caller order +// So process the string vector reversely +static std::string +extractPrefixContextId(const SmallVector &Probes, + ProfiledBinary *Binary) { + SmallVector ContextStrStack; + for (auto PI = Probes.rbegin(); PI != Probes.rend(); ++PI) { + Binary->getInlineContextForProbe(*PI, ContextStrStack, true); + } + std::ostringstream OContextStr; + for (auto CIter = ContextStrStack.rbegin(); CIter != ContextStrStack.rend(); + ++CIter) { + if (OContextStr.str().size()) + OContextStr << " @ "; + OContextStr << *CIter; + } + return OContextStr.str(); +} + +void PseudoProbeProfileGenerator::generateProfile() { + for (auto &BI : BinarySampleCounters) { + ProfiledBinary *Binary = BI.first; + for (auto &CI : BI.second) { + const ProbeBasedCtxKey *CtxKey = + dyn_cast(CI.first.getPtr()); + // PrefixContextId is the context id string except for the leaf probe's + // context, the final ContextId will be: + // ContextId = PrefixContextId + LeafContextId; + std::string PrefixContextId = + extractPrefixContextId(CtxKey->Probes, Binary); + // Fill in function body samples from probes, also infer caller's samples + // from callee's probe + populateBodySamplesWithProbes(CI.second.RangeCounter, PrefixContextId, + Binary); + // Fill in boudary samples for a call probe + populateBoundarySamplesWithProbes(CI.second.BranchCounter, + PrefixContextId, Binary); + } + } +} + +void PseudoProbeProfileGenerator::extractProbesFromRange( + const RangeSample &RangeCounter, ProbeCounterMap &ProbeCounter, + ProfiledBinary *Binary) { + RangeSample Ranges; + findDisjointRanges(Ranges, RangeCounter); + for (auto Range : Ranges) { + uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first); + uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second); + uint64_t Count = Range.second; + // Disjoint ranges have introduce zero-filled gap that + // doesn't belong to current context, filter them out. + if (Count == 0) + continue; + + InstructionPointer IP(Binary, RangeBegin, true); + + // Disjoint ranges may have range in the middle of two instr, + // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range + // can be Addr1+1 to Addr2-1. We should ignore such range. + if (IP.Address > RangeEnd) + continue; + + while (IP.Address <= RangeEnd) { + const AddressProbesMap &Address2ProbesMap = + Binary->getAddress2ProbesMap(); + auto It = Address2ProbesMap.find(IP.Address); + if (It != Address2ProbesMap.end()) { + for (auto &Probe : It->second) { + if (!Probe.isBlock()) + continue; + ProbeCounter[&Probe] += Count; + } + } + + IP.advance(); + } + } +} + +void PseudoProbeProfileGenerator::populateBodySamplesWithProbes( + const RangeSample &RangeCounter, StringRef PrefixContextId, + ProfiledBinary *Binary) { + ProbeCounterMap ProbeCounter; + // Extract the top frame probes by looking up each address among the range in + // the Address2ProbeMap + extractProbesFromRange(RangeCounter, ProbeCounter, Binary); + for (auto PI : ProbeCounter) { + const PseudoProbe *Probe = PI.first; + uint64_t Count = PI.second; + FunctionSamples &FunctionProfile = + getFunctionProfileForLeafProbe(PrefixContextId, Probe, Binary); + if (Probe->isDangling()) + Count = 0; + + FunctionProfile.addBodySamples(Probe->Index, 0, Count); + FunctionProfile.addTotalSamples(Count); + if (Probe->isEntry()) { + FunctionProfile.addHeadSamples(Count); + // Look up for the caller's functin profile + SmallVector LeafInlinedContext; + Binary->getInlineContextForProbe(Probe, LeafInlinedContext, false); + if (LeafInlinedContext.size()) { + StringRef CallerLoc = LeafInlinedContext[0]; + uint64_t CallerIndex = 0; + CallerLoc.split(":").second.getAsInteger(10, CallerIndex); + FunctionSamples &CallerProfile = + getFunctionProfileForLeaf(PrefixContextId, LeafInlinedContext); + CallerProfile.addBodySamples(CallerIndex, 0, Count); + CallerProfile.addTotalSamples(Count); + CallerProfile.addCalledTargetSamples(CallerIndex, 0, + FunctionProfile.getName(), Count); + } + } + } +} + +void PseudoProbeProfileGenerator::populateBoundarySamplesWithProbes( + const BranchSample &BranchCounter, StringRef PrefixContextId, + ProfiledBinary *Binary) { + for (auto BI : BranchCounter) { + uint64_t SourceOffset = BI.first.first; + uint64_t TargetOffset = BI.first.second; + uint64_t Count = BI.second; + StringRef CalleeName = FunctionSamples::getCanonicalFnName( + Binary->getFuncFromStartOffset(TargetOffset)); + if (CalleeName.size() == 0) + continue; + + uint64_t SourceAddress = Binary->offsetToVirtualAddr(SourceOffset); + const PseudoProbe *CallProbe = Binary->getCallProbeForAddr(SourceAddress); + if (CallProbe == nullptr) + continue; + FunctionSamples &FunctionProfile = + getFunctionProfileForLeafProbe(PrefixContextId, CallProbe, Binary); + FunctionProfile.addBodySamples(CallProbe->Index, 0, Count); + FunctionProfile.addTotalSamples(Count); + FunctionProfile.addCalledTargetSamples(CallProbe->Index, 0, CalleeName, + Count); + } +} + +FunctionSamples &PseudoProbeProfileGenerator::getFunctionProfileForLeaf( + StringRef PrefixContextId, + SmallVector &LeafInlinedContext) { + assert(LeafInlinedContext.size() && "Profile id must have the leaf frame"); + std::ostringstream OContextStr; + OContextStr << PrefixContextId.str(); + // For leaf inlined context with the top frame, we should strip off the top + // frame's probe id, like: + // Inlined stack: [bar:3, foo:2], the ContextId will be "foo:1 @ bar" + StringRef LeafLoc = LeafInlinedContext[0]; + StringRef LeafFuncName = LeafLoc.split(":").first; + for (int32_t I = LeafInlinedContext.size() - 1; I >= 0; I--) { + if (OContextStr.str().size()) + OContextStr << " @ "; + if (I) + OContextStr << LeafInlinedContext[I]; + else + OContextStr << LeafFuncName.str(); + } + FunctionSamples &FunctionProile = + getFunctionProfileForContext(OContextStr.str()); + return FunctionProile; +} + +FunctionSamples &PseudoProbeProfileGenerator::getFunctionProfileForLeafProbe( + StringRef PrefixContextId, const PseudoProbe *LeafProbe, + ProfiledBinary *Binary) { + // ContextId = PrefixContextId + LeafContextId; + // Use ContextId to look up FunctionProfile in ProfileMap + SmallVector LeafInlinedContext; + Binary->getInlineContextForProbe(LeafProbe, LeafInlinedContext, true); + return getFunctionProfileForLeaf(PrefixContextId, LeafInlinedContext); +} + } // end namespace sampleprof } // end namespace llvm diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h --- a/llvm/tools/llvm-profgen/ProfiledBinary.h +++ b/llvm/tools/llvm-profgen/ProfiledBinary.h @@ -248,6 +248,9 @@ return ProbeDecoder.getInlineContextForProbe(Probe, InlineContextStack, IncludeLeaf); } + const AddressProbesMap &getAddress2ProbesMap() const { + return ProbeDecoder.getAddress2ProbesMap(); + } }; } // end namespace sampleprof diff --git a/llvm/tools/llvm-profgen/PseudoProbe.h b/llvm/tools/llvm-profgen/PseudoProbe.h --- a/llvm/tools/llvm-profgen/PseudoProbe.h +++ b/llvm/tools/llvm-profgen/PseudoProbe.h @@ -199,12 +199,15 @@ // InlineContextStack Current leaf location info will be added if IncludeLeaf // is true Example: // Current probe(bar:3) inlined at foo:2 then inlined at main:1 - // IncludeLeaf = true, Output: [main:1, foo:2, bar:3] - // IncludeLeaf = false, OUtput: [main:1, foo:2] + // IncludeLeaf = true, Output: [bar:3, foo:2, main:1] + // IncludeLeaf = false, OUtput: [foo:2, main:1] void getInlineContextForProbe(const PseudoProbe *Probe, SmallVector &InlineContextStack, bool IncludeLeaf) const; + const AddressProbesMap &getAddress2ProbesMap() const { + return Address2ProbesMap; + } }; } // end namespace sampleprof