diff --git a/llvm/test/tools/llvm-profgen/inline-noprobe.test b/llvm/test/tools/llvm-profgen/inline-noprobe.test --- a/llvm/test/tools/llvm-profgen/inline-noprobe.test +++ b/llvm/test/tools/llvm-profgen/inline-noprobe.test @@ -4,10 +4,13 @@ ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK CHECK: main:836:0 +CHECK: 0: 0 +CHECK: 2: 0 CHECK: 1: foo:836 CHECK: 2.1: 42 CHECK: 3: 62 CHECK: 3.2: 21 +CHECK: 4: 0 CHECK: 3.1: bar:252 CHECK: 1: 42 CHECK: 3.2: bar:63 diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h --- a/llvm/tools/llvm-profgen/ProfileGenerator.h +++ b/llvm/tools/llvm-profgen/ProfileGenerator.h @@ -44,6 +44,7 @@ void write(); private: + void RangeSample preProcessRangeCounter(const RangeSample &RangeCounter); FunctionSamples &getTopLevelFunctionProfile(StringRef FuncName); // Helper function to get the leaf frame's FunctionProfile by traversing the // inline stack and meanwhile it adds the total samples for each frame's diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -146,11 +146,25 @@ // Sum of sample counts ending at this point uint64_t EndCount; - BoundaryPoint() : BeginCount(0), EndCount(0){}; + bool isZeroRangeBegin; - void addBeginCount(uint64_t Count) { BeginCount += Count; } + bool isZeroRangeEnd; - void addEndCount(uint64_t Count) { EndCount += Count; } + BoundaryPoint() + : BeginCount(UINT64_MAX), EndCount(UINT64_MAX), isZeroRangeBegin(false), + isZeroRangeEnd(false){}; + + void addBeginCount(uint64_t Count) { + if (BeginCount == UINT64_MAX) + BeginCount = 0; + BeginCount += Count; + } + + void addEndCount(uint64_t Count) { + if (EndCount == UINT64_MAX) + EndCount = 0; + EndCount += Count; + } }; /* @@ -179,6 +193,19 @@ [A, B-1]: 100 [B, B]: 300 [B+1, C]: 200. + + Example for zero value range: + + |<--- 100 --->| + |<--- 200 --->| + |<--------------- 0 ----------------->| + A B C D E F + + [A, B-1] : 0 + [B, C] : 100 + [C+1, D-1]: 0 + [D, E] : 200 + [E+1, F] : 0 */ std::map Boundaries; @@ -193,25 +220,35 @@ if (Boundaries.find(End) == Boundaries.end()) Boundaries[End] = BoundaryPoint(); Boundaries[End].addEndCount(Count); + + if (Count == 0) { + Boundaries[Begin].isZeroRangeBegin = true; + Boundaries[End].isZeroRangeEnd = true; + } } uint64_t BeginAddress = UINT64_MAX; + int ZeroRangeDepth = 0; int Count = 0; for (auto Item : Boundaries) { uint64_t Address = Item.first; BoundaryPoint &Point = Item.second; - if (Point.BeginCount) { + if (Point.BeginCount != UINT64_MAX) { if (BeginAddress != UINT64_MAX) DisjointRanges[{BeginAddress, Address - 1}] = Count; Count += Point.BeginCount; BeginAddress = Address; + ZeroRangeDepth += Point.isZeroRangeBegin; } - if (Point.EndCount) { + if (Point.EndCount != UINT64_MAX) { assert((BeginAddress != UINT64_MAX) && "First boundary point cannot be 'end' point"); DisjointRanges[{BeginAddress, Address}] = Count; Count -= Point.EndCount; BeginAddress = Address + 1; + ZeroRangeDepth -= Point.isZeroRangeEnd; + if (Count == 0 && ZeroRangeDepth == 0) + BeginAddress = UINT64_MAX; } } } @@ -260,18 +297,37 @@ return *FunctionProfile; } +void RangeSample +ProfileGenerator::preProcessRangeCounter(const RangeSample &RangeCounter) { + RangeSample Ranges(RangeCounter.begin(), RangeCounter.end()); + // For each range, we search for the range of the function it belongs to and + // initialize it with zero count, so it remains zero if doesn't hit any + // samples. This is to be consistent with compiler that interpret zero count + // as unexecuted(cold). + for (auto I : RangeCounter) { + uint64_t RangeBegin = I.first.first; + uint64_t RangeEnd = I.first.second; + // Find the function offset range the current range begin belongs to. + auto FuncRange = Binary->findFuncOffsetRange(RangeBegin); + if (FuncRange.second == 0) + WithColor::warning() + << "Invalid range or disassembling error in profiled binary.\n"; + else if (RangeEnd > FuncRange.second) + WithColor::warning() << "Range is across different functions.\n"; + else + Ranges[FuncRange] += 0; + } + RangeSample DisjointRanges; + findDisjointRanges(DisjointRanges, Ranges); + return DisjointRanges; +} + void ProfileGenerator::populateFunctionBodySamples( const RangeSample &RangeCounter) { - RangeSample Ranges; - findDisjointRanges(Ranges, RangeCounter); - for (auto Range : Ranges) { + for (auto Range : preProcessRangeCounter(RangeCounter)) { uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first); uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second); uint64_t Count = Range.second; - // Disjoint ranges have introduce zero-filled gap that - // doesn't belong to current context, filter them out. - if (Count == 0) - continue; InstructionPointer IP(Binary, RangeBegin, true); // Disjoint ranges may have range in the middle of two instr, @@ -370,8 +426,9 @@ // Use the maximum count of samples with same line location ErrorOr R = FunctionProfile.findSamplesAt( LeafLoc.Callsite.LineOffset, LeafLoc.Callsite.Discriminator); + uint64_t PreviousCount = R ? R.get() : 0; - if (PreviousCount < Count) { + if (PreviousCount <= Count) { FunctionProfile.addBodySamples(LeafLoc.Callsite.LineOffset, LeafLoc.Callsite.Discriminator, Count - PreviousCount); diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h --- a/llvm/tools/llvm-profgen/ProfiledBinary.h +++ b/llvm/tools/llvm-profgen/ProfiledBinary.h @@ -31,6 +31,7 @@ #include "llvm/Support/Path.h" #include "llvm/Transforms/IPO/SampleContextTracker.h" #include +#include #include #include #include @@ -79,7 +80,7 @@ // Take the two addresses from the start of function as prolog void inferPrologOffsets( - std::unordered_map &FuncStartAddrMap) { + std::map> &FuncStartAddrMap) { for (auto I : FuncStartAddrMap) { PrologEpilogSet.insert(I.first); InstructionPointer IP(Binary, I.first); @@ -138,6 +139,8 @@ ContextTrieNode RootContext; }; +using OffsetRange = std::pair; + class ProfiledBinary { // Absolute path of the binary. std::string Path; @@ -162,7 +165,7 @@ // if a given RVA is a valid code address. std::set> TextSections; // Function offset to name mapping. - std::unordered_map FuncStartAddrMap; + std::map> FuncStartAddrMap; // Offset to context location map. Used to expand the context. std::unordered_map Offset2LocStackMap; // An array of offsets of all instructions sorted in increasing order. The @@ -300,7 +303,15 @@ auto I = FuncStartAddrMap.find(Offset); if (I == FuncStartAddrMap.end()) return StringRef(); - return I->second; + return I->second.first; + } + + OffsetRange findFuncOffsetRange(uint64_t Offset) { + auto I = FuncStartAddrMap.upper_bound(Offset); + if (I == FuncStartAddrMap.begin()) + return {0, 0}; + I--; + return {I->first, I->second.second}; } uint32_t getFuncSizeForContext(SampleContext &Context) { diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp --- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp +++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp @@ -388,7 +388,7 @@ if (ShowDisassemblyOnly) outs() << "\n"; - FuncStartAddrMap[StartOffset] = Symbols[SI].Name.str(); + FuncStartAddrMap[StartOffset] = {Symbols[SI].Name.str(), EndOffset}; return true; }