diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h --- a/llvm/include/llvm/MC/MCPseudoProbe.h +++ b/llvm/include/llvm/MC/MCPseudoProbe.h @@ -45,9 +45,25 @@ #define LLVM_MC_MCPSEUDOPROBE_H #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/PseudoProbe.h" #include "llvm/MC/MCSection.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" +#include #include +#include #include +#include +#include +#include +#include +#include +#include #include namespace llvm { @@ -62,69 +78,213 @@ AddressDelta = 0x1, }; +// Function descriptor decoded from .pseudo_probe_desc section +struct MCPseudoProbeFuncDesc { + uint64_t FuncGUID = 0; + uint64_t FuncHash = 0; + std::string FuncName; + + MCPseudoProbeFuncDesc(uint64_t GUID, uint64_t Hash, StringRef Name) + : FuncGUID(GUID), FuncHash(Hash), FuncName(Name){}; + + void print(raw_ostream &OS); +}; + +class MCPseudoProbe; +class MCDecodedPseudoProbe; + +// An inline frame has the form +using InlineSite = std::tuple; +using MCPseudoProbeInlineStack = SmallVector; +// GUID to PseudoProbeFuncDesc map +using GUIDProbeFunctionMap = + std::unordered_map; +// Address to pseudo probes map. +using AddressProbesMap = + std::unordered_map>; + +class MCPseudoProbeInlineTree; +class MCDecodedPseudoProbeInlineTree; + +class MCPseudoProbeBase { +protected: + uint64_t Guid; + uint64_t Index; + uint8_t Attributes; + uint8_t Type; + // The value should be equal to PseudoProbeReservedId::Last + 1 which is + // defined in SampleProfileProbe.h. The header file is not included here to + // reduce the dependency from MC to IPO. + const static uint32_t PseudoProbeFirstId = 1; + +public: + MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T) + : Guid(G), Index(I), Attributes(At), Type(T) {} + + bool isEntry() const { return Index == PseudoProbeFirstId; } + + bool isTailCall() const { + return Attributes & static_cast(PseudoProbeAttributes::Reserved); + } + + uint64_t getGuid() const { return Guid; } + + uint64_t getIndex() const { return Index; } + + uint8_t getAttributes() const { return Attributes; } + + uint8_t getType() const { return Type; } + + bool isBlock() const { + return Type == static_cast(PseudoProbeType::Block); + } + + bool isIndirectCall() const { + return Type == static_cast(PseudoProbeType::IndirectCall); + } + + bool isDirectCall() const { + return Type == static_cast(PseudoProbeType::DirectCall); + } + + bool isCall() const { return isIndirectCall() || isDirectCall(); } + + void setAttributes(uint8_t Attr) { Attributes = Attr; } +}; + /// Instances of this class represent a pseudo probe instance for a pseudo probe /// table entry, which is created during a machine instruction is assembled and /// uses an address from a temporary label created at the current address in the /// current section. -class MCPseudoProbe { +class MCPseudoProbe : public MCPseudoProbeBase { MCSymbol *Label; - uint64_t Guid; - uint64_t Index; - uint8_t Type; - uint8_t Attributes; public: MCPseudoProbe(MCSymbol *Label, uint64_t Guid, uint64_t Index, uint64_t Type, uint64_t Attributes) - : Label(Label), Guid(Guid), Index(Index), Type(Type), - Attributes(Attributes) { + : MCPseudoProbeBase(Guid, Index, Attributes, Type), Label(Label) { assert(Type <= 0xFF && "Probe type too big to encode, exceeding 2^8"); assert(Attributes <= 0xFF && "Probe attributes too big to encode, exceeding 2^16"); } MCSymbol *getLabel() const { return Label; } + void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *LastProbe) const; +}; - uint64_t getGuid() const { return Guid; } +class MCDecodedPseudoProbe : public MCPseudoProbeBase { + uint64_t Address; + MCDecodedPseudoProbeInlineTree *InlineTree; - uint64_t getIndex() const { return Index; } +public: + MCDecodedPseudoProbe(uint64_t Ad, uint64_t G, uint32_t I, PseudoProbeType K, + uint8_t At, MCDecodedPseudoProbeInlineTree *Tree) + : MCPseudoProbeBase(G, I, At, static_cast(K)), Address(Ad), + InlineTree(Tree){}; - uint8_t getType() const { return Type; } + uint64_t getAddress() const { return Address; } - uint8_t getAttributes() const { return Attributes; } + void setAddress(uint64_t Addr) { Address = Addr; } - void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *LastProbe) const; + MCDecodedPseudoProbeInlineTree *getInlineTreeNode() const { + return InlineTree; + } + + // Get the inlined context by traversing current inline tree backwards, + // each tree node has its InlineSite which is taken as the context. + // \p ContextStack is populated in root to leaf order + void getInlineContext(SmallVectorImpl &ContextStack, + const GUIDProbeFunctionMap &GUID2FuncMAP, + bool ShowName) const; + + // Helper function to get the string from context stack + std::string getInlineContextStr(const GUIDProbeFunctionMap &GUID2FuncMAP, + bool ShowName) const; + + // Print pseudo probe while disassembling + void print(raw_ostream &OS, const GUIDProbeFunctionMap &GUID2FuncMAP, + bool ShowName) const; }; -// An inline frame has the form -using InlineSite = std::tuple; -using MCPseudoProbeInlineStack = SmallVector; +template +class MCPseudoProbeInlineTreeBase { + struct InlineSiteHash { + uint64_t operator()(const InlineSite &Site) const { + return std::get<0>(Site) ^ std::get<1>(Site); + } + }; + +protected: + // Track children (e.g. inlinees) of current context + using InlinedProbeTreeMap = std::unordered_map< + InlineSite, std::unique_ptr, InlineSiteHash>; + InlinedProbeTreeMap Children; + // Set of probes that come with the function. + std::vector Probes; + MCPseudoProbeInlineTreeBase() { + static_assert(std::is_base_of::value, + "DerivedProbeInlineTreeType must be subclass of " + "MCPseudoProbeInlineTreeBase"); + } + +public: + uint64_t Guid = 0; + + // Root node has a GUID 0. + bool isRoot() const { return Guid == 0; } + InlinedProbeTreeMap &getChildren() { return Children; } + const InlinedProbeTreeMap &getChildren() const { return Children; } + std::vector &getProbes() { return Probes; } + void addProbes(ProbeType Probe) { Probes.push_back(Probe); } + // Caller node of the inline site + MCPseudoProbeInlineTreeBase *Parent; + DerivedProbeInlineTreeType *getOrAddNode(const InlineSite &Site) { + auto Ret = Children.emplace( + Site, std::make_unique(Site)); + Ret.first->second->Parent = this; + return Ret.first->second.get(); + }; +}; // A Tri-tree based data structure to group probes by inline stack. // A tree is allocated for a standalone .text section. A fake // instance is created as the root of a tree. // A real instance of this class is created for each function, either an // unlined function that has code in .text section or an inlined function. -class MCPseudoProbeInlineTree { - uint64_t Guid; - // Set of probes that come with the function. - std::vector Probes; - // Use std::map for a deterministic output. - std::map Inlinees; - - // Root node has a GUID 0. - bool isRoot() { return Guid == 0; } - MCPseudoProbeInlineTree *getOrAddNode(InlineSite Site); +class MCPseudoProbeInlineTree + : public MCPseudoProbeInlineTreeBase { public: MCPseudoProbeInlineTree() = default; - MCPseudoProbeInlineTree(uint64_t Guid) : Guid(Guid) {} - ~MCPseudoProbeInlineTree(); + MCPseudoProbeInlineTree(uint64_t Guid) { this->Guid = Guid; } + MCPseudoProbeInlineTree(const InlineSite &Site) { + this->Guid = std::get<0>(Site); + } + + // MCPseudoProbeInlineTree method based on Inlinees void addPseudoProbe(const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack); void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *&LastProbe); }; +// inline tree node for the decoded pseudo probe +class MCDecodedPseudoProbeInlineTree + : public MCPseudoProbeInlineTreeBase { +public: + InlineSite ISite; + // Used for decoding + uint32_t ChildrenToProcess = 0; + + MCDecodedPseudoProbeInlineTree(){}; + MCDecodedPseudoProbeInlineTree(const InlineSite &Site) : ISite(Site){}; + + // Return false if it's a dummy inline site + bool hasInlineSite() const { return std::get<0>(ISite) != 0; } +}; + /// Instances of this class represent the pseudo probes inserted into a compile /// unit. class MCPseudoProbeSection { @@ -172,6 +332,83 @@ static int DdgPrintIndent; #endif }; + +class MCPseudoProbeDecoder { + // GUID to PseudoProbeFuncDesc map. + GUIDProbeFunctionMap GUID2FuncDescMap; + + // Address to probes map. + AddressProbesMap Address2ProbesMap; + + // The dummy root of the inline trie, all the outlined function will directly + // be the children of the dummy root, all the inlined function will be the + // children of its inlineer. So the relation would be like: + // DummyRoot --> OutlinedFunc --> InlinedFunc1 --> InlinedFunc2 + MCDecodedPseudoProbeInlineTree DummyInlineRoot; + + /// Points to the current location in the buffer. + const uint8_t *Data = nullptr; + + /// Points to the end of the buffer. + const uint8_t *End = nullptr; + + // Decoding helper function + template ErrorOr readUnencodedNumber(); + template ErrorOr readUnsignedNumber(); + template ErrorOr readSignedNumber(); + ErrorOr readString(uint32_t Size); + +public: + // Decode pseudo_probe_desc section to build GUID to PseudoProbeFuncDesc map. + bool buildGUID2FuncDescMap(const uint8_t *Start, std::size_t Size); + + // Decode pseudo_probe section to build address to probes map. + bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size); + + // Print pseudo_probe_desc section info + void printGUID2FuncDescMap(raw_ostream &OS); + + // Print pseudo_probe section info, used along with show-disassembly + void printProbeForAddress(raw_ostream &OS, uint64_t Address); + + // do printProbeForAddress for all addresses + void printProbesForAllAddresses(raw_ostream &OS); + + // Look up the probe of a call for the input address + const MCDecodedPseudoProbe *getCallProbeForAddr(uint64_t Address) const; + + const MCPseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) const; + + // Helper function to populate one probe's inline stack into + // \p InlineContextStack. + // Current leaf location info will be added if IncludeLeaf is true + // Example: + // Current probe(bar:3) inlined at foo:2 then inlined at main:1 + // IncludeLeaf = true, Output: [main:1, foo:2, bar:3] + // IncludeLeaf = false, Output: [main:1, foo:2] + void + getInlineContextForProbe(const MCDecodedPseudoProbe *Probe, + SmallVectorImpl &InlineContextStack, + bool IncludeLeaf) const; + + const AddressProbesMap &getAddress2ProbesMap() const { + return Address2ProbesMap; + } + + AddressProbesMap &getAddress2ProbesMap() { return Address2ProbesMap; } + + const GUIDProbeFunctionMap &getGUID2FuncDescMap() const { + return GUID2FuncDescMap; + } + + const MCPseudoProbeFuncDesc * + getInlinerDescForProbe(const MCDecodedPseudoProbe *Probe) const; + + const MCDecodedPseudoProbeInlineTree &getDummyInlineRoot() const { + return DummyInlineRoot; + } +}; + } // end namespace llvm #endif // LLVM_MC_MCPSEUDOPROBE_H diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt --- a/llvm/lib/MC/CMakeLists.txt +++ b/llvm/lib/MC/CMakeLists.txt @@ -70,6 +70,7 @@ Support BinaryFormat DebugInfoCodeView + ProfileData ) add_subdirectory(MCParser) diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp --- a/llvm/lib/MC/MCPseudoProbe.cpp +++ b/llvm/lib/MC/MCPseudoProbe.cpp @@ -12,10 +12,18 @@ #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCStreamer.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/LEB128.h" +#include "llvm/Support/raw_ostream.h" +#include +#include #define DEBUG_TYPE "mcpseudoprobe" using namespace llvm; +using namespace support; +using namespace sampleprof; #ifndef NDEBUG int MCPseudoProbeTable::DdgPrintIndent = 0; @@ -69,23 +77,6 @@ }); } -MCPseudoProbeInlineTree::~MCPseudoProbeInlineTree() { - for (auto &Inlinee : Inlinees) - delete Inlinee.second; -} - -MCPseudoProbeInlineTree * -MCPseudoProbeInlineTree::getOrAddNode(InlineSite Site) { - auto Iter = Inlinees.find(Site); - if (Iter == Inlinees.end()) { - auto *Node = new MCPseudoProbeInlineTree(std::get<0>(Site)); - Inlinees[Site] = Node; - return Node; - } else { - return Iter->second; - } -} - void MCPseudoProbeInlineTree::addPseudoProbe( const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) { // The function should not be called on the root. @@ -147,7 +138,7 @@ // Emit number of probes in this node MCOS->emitULEB128IntValue(Probes.size()); // Emit number of direct inlinees - MCOS->emitULEB128IntValue(Inlinees.size()); + MCOS->emitULEB128IntValue(Children.size()); // Emit probes in this group for (const auto &Probe : Probes) { Probe.emit(MCOS, LastProbe); @@ -157,7 +148,13 @@ assert(Probes.empty() && "Root should not have probes"); } - // Emit descendent + // Emit sorted descendant + // InlineSite is unique for each pair, + // so there will be no ordering of Inlinee based on MCPseudoProbeInlineTree* + std::map Inlinees; + for (auto Child = Children.begin(); Child != Children.end(); ++Child) + Inlinees[Child->first] = Child->second.get(); + for (const auto &Inlinee : Inlinees) { if (Guid) { // Emit probe index @@ -211,3 +208,371 @@ // Put out the probe. ProbeSections.emit(MCOS); } + +static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP, + uint64_t GUID) { + auto It = GUID2FuncMAP.find(GUID); + assert(It != GUID2FuncMAP.end() && + "Probe function must exist for a valid GUID"); + return It->second.FuncName; +} + +void MCPseudoProbeFuncDesc::print(raw_ostream &OS) { + OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n"; + OS << "Hash: " << FuncHash << "\n"; +} + +void MCDecodedPseudoProbe::getInlineContext( + SmallVectorImpl &ContextStack, + const GUIDProbeFunctionMap &GUID2FuncMAP, bool ShowName) const { + uint32_t Begin = ContextStack.size(); + MCDecodedPseudoProbeInlineTree *Cur = InlineTree; + // It will add the string of each node's inline site during iteration. + // Note that it won't include the probe's belonging function(leaf location) + while (Cur->hasInlineSite()) { + std::string ContextStr; + if (ShowName) { + StringRef FuncName = + getProbeFNameForGUID(GUID2FuncMAP, std::get<0>(Cur->ISite)); + ContextStr += FuncName.str(); + } else { + ContextStr += Twine(std::get<0>(Cur->ISite)).str(); + } + ContextStr += ":"; + ContextStr += Twine(std::get<1>(Cur->ISite)).str(); + ContextStack.emplace_back(ContextStr); + Cur = static_cast(Cur->Parent); + } + // Make the ContextStack in caller-callee order + std::reverse(ContextStack.begin() + Begin, ContextStack.end()); +} + +std::string MCDecodedPseudoProbe::getInlineContextStr( + const GUIDProbeFunctionMap &GUID2FuncMAP, bool ShowName) const { + std::ostringstream OContextStr; + SmallVector ContextStack; + getInlineContext(ContextStack, GUID2FuncMAP, ShowName); + for (auto &CxtStr : ContextStack) { + if (OContextStr.str().size()) + OContextStr << " @ "; + OContextStr << CxtStr; + } + return OContextStr.str(); +} + +static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall", + "DirectCall"}; + +void MCDecodedPseudoProbe::print(raw_ostream &OS, + const GUIDProbeFunctionMap &GUID2FuncMAP, + bool ShowName) const { + OS << "FUNC: "; + if (ShowName) { + StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Guid); + OS << FuncName.str() << " "; + } else { + OS << Guid << " "; + } + OS << "Index: " << Index << " "; + OS << "Type: " << PseudoProbeTypeStr[static_cast(Type)] << " "; + if (isTailCall()) { + OS << "TailCall "; + } + std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP, ShowName); + if (InlineContextStr.size()) { + OS << "Inlined: @ "; + OS << InlineContextStr; + } + OS << "\n"; +} + +template ErrorOr MCPseudoProbeDecoder::readUnencodedNumber() { + if (Data + sizeof(T) > End) { + return std::error_code(); + } + T Val = endian::readNext(Data); + return ErrorOr(Val); +} + +template ErrorOr MCPseudoProbeDecoder::readUnsignedNumber() { + unsigned NumBytesRead = 0; + uint64_t Val = decodeULEB128(Data, &NumBytesRead); + if (Val > std::numeric_limits::max() || (Data + NumBytesRead > End)) { + return std::error_code(); + } + Data += NumBytesRead; + return ErrorOr(static_cast(Val)); +} + +template ErrorOr MCPseudoProbeDecoder::readSignedNumber() { + unsigned NumBytesRead = 0; + int64_t Val = decodeSLEB128(Data, &NumBytesRead); + if (Val > std::numeric_limits::max() || (Data + NumBytesRead > End)) { + return std::error_code(); + } + Data += NumBytesRead; + return ErrorOr(static_cast(Val)); +} + +ErrorOr MCPseudoProbeDecoder::readString(uint32_t Size) { + StringRef Str(reinterpret_cast(Data), Size); + if (Data + Size > End) { + return std::error_code(); + } + Data += Size; + return ErrorOr(Str); +} + +bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start, + std::size_t Size) { + // The pseudo_probe_desc section has a format like: + // .section .pseudo_probe_desc,"",@progbits + // .quad -5182264717993193164 // GUID + // .quad 4294967295 // Hash + // .uleb 3 // Name size + // .ascii "foo" // Name + // .quad -2624081020897602054 + // .quad 174696971957 + // .uleb 34 + // .ascii "main" + + Data = Start; + End = Data + Size; + + while (Data < End) { + auto ErrorOrGUID = readUnencodedNumber(); + if (!ErrorOrGUID) + return false; + + auto ErrorOrHash = readUnencodedNumber(); + if (!ErrorOrHash) + return false; + + auto ErrorOrNameSize = readUnsignedNumber(); + if (!ErrorOrNameSize) + return false; + uint32_t NameSize = std::move(*ErrorOrNameSize); + + auto ErrorOrName = readString(NameSize); + if (!ErrorOrName) + return false; + + uint64_t GUID = std::move(*ErrorOrGUID); + uint64_t Hash = std::move(*ErrorOrHash); + StringRef Name = + FunctionSamples::getCanonicalFnName(std::move(*ErrorOrName)); + + // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap + GUID2FuncDescMap.emplace(GUID, MCPseudoProbeFuncDesc(GUID, Hash, Name)); + } + assert(Data == End && "Have unprocessed data in pseudo_probe_desc section"); + return true; +} + +bool MCPseudoProbeDecoder::buildAddress2ProbeMap(const uint8_t *Start, + std::size_t Size) { + // The pseudo_probe section encodes an inline forest and each tree has a + // format like: + // FUNCTION BODY (one for each uninlined function present in the text + // section) + // GUID (uint64) + // GUID of the function + // NPROBES (ULEB128) + // Number of probes originating from this function. + // NUM_INLINED_FUNCTIONS (ULEB128) + // Number of callees inlined into this function, aka number of + // first-level inlinees + // PROBE RECORDS + // A list of NPROBES entries. Each entry contains: + // INDEX (ULEB128) + // TYPE (uint4) + // 0 - block probe, 1 - indirect call, 2 - direct call + // ATTRIBUTE (uint3) + // 1 - tail call, 2 - dangling + // ADDRESS_TYPE (uint1) + // 0 - code address, 1 - address delta + // CODE_ADDRESS (uint64 or ULEB128) + // code address or address delta, depending on Flag + // INLINED FUNCTION RECORDS + // A list of NUM_INLINED_FUNCTIONS entries describing each of the + // inlined callees. Each record contains: + // INLINE SITE + // Index of the callsite probe (ULEB128) + // FUNCTION BODY + // A FUNCTION BODY entry describing the inlined function. + + Data = Start; + End = Data + Size; + + MCDecodedPseudoProbeInlineTree *Root = &DummyInlineRoot; + MCDecodedPseudoProbeInlineTree *Cur = &DummyInlineRoot; + uint64_t LastAddr = 0; + uint32_t Index = 0; + // A DFS-based decoding + while (Data < End) { + if (Root == Cur) { + // Use a sequential id for top level inliner. + Index = Root->getChildren().size(); + } else { + // Read inline site for inlinees + auto ErrorOrIndex = readUnsignedNumber(); + if (!ErrorOrIndex) + return false; + Index = std::move(*ErrorOrIndex); + } + // Switch/add to a new tree node(inlinee) + Cur = Cur->getOrAddNode(std::make_tuple(Cur->Guid, Index)); + // Read guid + auto ErrorOrCurGuid = readUnencodedNumber(); + if (!ErrorOrCurGuid) + return false; + Cur->Guid = std::move(*ErrorOrCurGuid); + // Read number of probes in the current node. + auto ErrorOrNodeCount = readUnsignedNumber(); + if (!ErrorOrNodeCount) + return false; + uint32_t NodeCount = std::move(*ErrorOrNodeCount); + // Read number of direct inlinees + auto ErrorOrCurChildrenToProcess = readUnsignedNumber(); + if (!ErrorOrCurChildrenToProcess) + return false; + Cur->ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess); + // Read all probes in this node + for (std::size_t I = 0; I < NodeCount; I++) { + // Read index + auto ErrorOrIndex = readUnsignedNumber(); + if (!ErrorOrIndex) + return false; + uint32_t Index = std::move(*ErrorOrIndex); + // Read type | flag. + auto ErrorOrValue = readUnencodedNumber(); + if (!ErrorOrValue) + return false; + uint8_t Value = std::move(*ErrorOrValue); + uint8_t Kind = Value & 0xf; + uint8_t Attr = (Value & 0x70) >> 4; + // Read address + uint64_t Addr = 0; + if (Value & 0x80) { + auto ErrorOrOffset = readSignedNumber(); + if (!ErrorOrOffset) + return false; + int64_t Offset = std::move(*ErrorOrOffset); + Addr = LastAddr + Offset; + } else { + auto ErrorOrAddr = readUnencodedNumber(); + if (!ErrorOrAddr) + return false; + Addr = std::move(*ErrorOrAddr); + } + // Populate Address2ProbesMap + auto &Probes = Address2ProbesMap[Addr]; + Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr, + Cur); + Cur->addProbes(&Probes.back()); + LastAddr = Addr; + } + + // Look for the parent for the next node by subtracting the current + // node count from tree counts along the parent chain. The first node + // in the chain that has a non-zero tree count is the target. + while (Cur != Root) { + if (Cur->ChildrenToProcess == 0) { + Cur = static_cast(Cur->Parent); + if (Cur != Root) { + assert(Cur->ChildrenToProcess > 0 && + "Should have some unprocessed nodes"); + Cur->ChildrenToProcess -= 1; + } + } else { + break; + } + } + } + + assert(Data == End && "Have unprocessed data in pseudo_probe section"); + assert(Cur == Root && + " Cur should point to root when the forest is fully built up"); + return true; +} + +void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) { + OS << "Pseudo Probe Desc:\n"; + // Make the output deterministic + std::map OrderedMap(GUID2FuncDescMap.begin(), + GUID2FuncDescMap.end()); + for (auto &I : OrderedMap) { + I.second.print(OS); + } +} + +void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS, + uint64_t Address) { + auto It = Address2ProbesMap.find(Address); + if (It != Address2ProbesMap.end()) { + for (auto &Probe : It->second) { + OS << " [Probe]:\t"; + Probe.print(OS, GUID2FuncDescMap, true); + } + } +} + +void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) { + std::vector Addresses; + for (auto Entry : Address2ProbesMap) + Addresses.push_back(Entry.first); + std::sort(Addresses.begin(), Addresses.end()); + for (auto K : Addresses) { + OS << "Address:\t"; + OS << K; + OS << "\n"; + printProbeForAddress(OS, K); + } +} + +const MCDecodedPseudoProbe * +MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const { + auto It = Address2ProbesMap.find(Address); + if (It == Address2ProbesMap.end()) + return nullptr; + const auto &Probes = It->second; + + const MCDecodedPseudoProbe *CallProbe = nullptr; + for (const auto &Probe : Probes) { + if (Probe.isCall()) { + assert(!CallProbe && + "There should be only one call probe corresponding to address " + "which is a callsite."); + CallProbe = &Probe; + } + } + return CallProbe; +} + +const MCPseudoProbeFuncDesc * +MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const { + auto It = GUID2FuncDescMap.find(GUID); + assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist"); + return &It->second; +} + +void MCPseudoProbeDecoder::getInlineContextForProbe( + const MCDecodedPseudoProbe *Probe, + SmallVectorImpl &InlineContextStack, bool IncludeLeaf) const { + Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap, true); + if (!IncludeLeaf) + return; + // Note that the context from probe doesn't include leaf frame, + // hence we need to retrieve and prepend leaf if requested. + const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid()); + InlineContextStack.emplace_back(FuncDesc->FuncName + ":" + + Twine(Probe->getIndex()).str()); +} + +const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe( + const MCDecodedPseudoProbe *Probe) const { + MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode(); + if (!InlinerNode->hasInlineSite()) + return nullptr; + return getFuncDescForGUID(std::get<0>(InlinerNode->ISite)); +} diff --git a/llvm/tools/llvm-profgen/CMakeLists.txt b/llvm/tools/llvm-profgen/CMakeLists.txt --- a/llvm/tools/llvm-profgen/CMakeLists.txt +++ b/llvm/tools/llvm-profgen/CMakeLists.txt @@ -19,5 +19,4 @@ CSPreInliner.cpp ProfiledBinary.cpp ProfileGenerator.cpp - PseudoProbe.cpp ) diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h --- a/llvm/tools/llvm-profgen/PerfReader.h +++ b/llvm/tools/llvm-profgen/PerfReader.h @@ -366,7 +366,7 @@ // need to be splitted by '@' to get the last location frame, so we // can just use probe instead and generate the string in the end. struct ProbeBasedCtxKey : public ContextKey { - SmallVector Probes; + SmallVector Probes; ProbeBasedCtxKey() : ContextKey(CK_ProbeBased) {} static bool classof(const ContextKey *K) { @@ -432,11 +432,12 @@ }; struct ProbeStack { - SmallVector Stack; + SmallVector Stack; const ProfiledBinary *Binary; ProbeStack(const ProfiledBinary *B) : Binary(B) {} bool pushFrame(UnwindState::ProfiledFrame *Cur) { - const PseudoProbe *CallProbe = Binary->getCallProbeForAddr(Cur->Address); + const MCDecodedPseudoProbe *CallProbe = + Binary->getCallProbeForAddr(Cur->Address); // We may not find a probe for a merged or external callsite. // Callsite merging may cause the loss of original probe IDs. // Cutting off the context from here since the inliner will diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp --- a/llvm/tools/llvm-profgen/PerfReader.cpp +++ b/llvm/tools/llvm-profgen/PerfReader.cpp @@ -107,7 +107,7 @@ for (auto CallProbe : Stack) { ProbeBasedKey->Probes.emplace_back(CallProbe); } - CSProfileGenerator::compressRecursionContext( + CSProfileGenerator::compressRecursionContext( ProbeBasedKey->Probes); ProbeBasedKey->genHashCode(); return ProbeBasedKey; diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h --- a/llvm/tools/llvm-profgen/ProfileGenerator.h +++ b/llvm/tools/llvm-profgen/ProfileGenerator.h @@ -214,7 +214,8 @@ static int32_t MaxCompressionSize; }; -using ProbeCounterMap = std::unordered_map; +using ProbeCounterMap = + std::unordered_map; class PseudoProbeCSProfileGenerator : public CSProfileGenerator { @@ -241,12 +242,12 @@ // Helper function to get FunctionSamples for the leaf inlined context FunctionSamples & getFunctionProfileForLeafProbe(SmallVectorImpl &ContextStrStack, - const PseudoProbeFuncDesc *LeafFuncDesc, + const MCPseudoProbeFuncDesc *LeafFuncDesc, bool WasLeafInlined); // Helper function to get FunctionSamples for the leaf probe FunctionSamples & getFunctionProfileForLeafProbe(SmallVectorImpl &ContextStrStack, - const PseudoProbe *LeafProbe, + const MCDecodedPseudoProbe *LeafProbe, ProfiledBinary *Binary); }; diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -443,8 +443,8 @@ // be added compressed while looking up function profile static void extractPrefixContextStack(SmallVectorImpl &ContextStrStack, - const SmallVectorImpl &Probes, - ProfiledBinary *Binary) { + const SmallVectorImpl &Probes, + ProfiledBinary *Binary) { for (const auto *P : Probes) { Binary->getInlineContextForProbe(P, ContextStrStack, true); } @@ -520,9 +520,10 @@ // Extract the top frame probes by looking up each address among the range in // the Address2ProbeMap extractProbesFromRange(RangeCounter, ProbeCounter, Binary); - std::unordered_map FrameSamples; + std::unordered_map + FrameSamples; for (auto PI : ProbeCounter) { - const PseudoProbe *Probe = PI.first; + const MCDecodedPseudoProbe *Probe = PI.first; uint64_t Count = PI.second; FunctionSamples &FunctionProfile = getFunctionProfileForLeafProbe(ContextStrStack, Probe, Binary); @@ -530,7 +531,7 @@ // collected for non-danglie probes. This is for reporting all of the // zero count probes of the frame later. FrameSamples[Probe->getInlineTreeNode()] = &FunctionProfile; - FunctionProfile.addBodySamplesForProbe(Probe->Index, Count); + FunctionProfile.addBodySamplesForProbe(Probe->getIndex(), Count); FunctionProfile.addTotalSamples(Count); if (Probe->isEntry()) { FunctionProfile.addHeadSamples(Count); @@ -565,7 +566,7 @@ for (auto &I : FrameSamples) { auto *FunctionProfile = I.second; for (auto *Probe : I.first->getProbes()) { - FunctionProfile->addBodySamplesForProbe(Probe->Index, 0); + FunctionProfile->addBodySamplesForProbe(Probe->getIndex(), 0); } } } @@ -579,25 +580,26 @@ uint64_t TargetOffset = BI.first.second; uint64_t Count = BI.second; uint64_t SourceAddress = Binary->offsetToVirtualAddr(SourceOffset); - const PseudoProbe *CallProbe = Binary->getCallProbeForAddr(SourceAddress); + const MCDecodedPseudoProbe *CallProbe = + Binary->getCallProbeForAddr(SourceAddress); if (CallProbe == nullptr) continue; FunctionSamples &FunctionProfile = getFunctionProfileForLeafProbe(ContextStrStack, CallProbe, Binary); - FunctionProfile.addBodySamples(CallProbe->Index, 0, Count); + FunctionProfile.addBodySamples(CallProbe->getIndex(), 0, Count); FunctionProfile.addTotalSamples(Count); StringRef CalleeName = FunctionSamples::getCanonicalFnName( Binary->getFuncFromStartOffset(TargetOffset)); if (CalleeName.size() == 0) continue; - FunctionProfile.addCalledTargetSamples(CallProbe->Index, 0, CalleeName, + FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(), 0, CalleeName, Count); } } FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe( SmallVectorImpl &ContextStrStack, - const PseudoProbeFuncDesc *LeafFuncDesc, bool WasLeafInlined) { + const MCPseudoProbeFuncDesc *LeafFuncDesc, bool WasLeafInlined) { assert(ContextStrStack.size() && "Profile context must have the leaf frame"); // Compress the context string except for the leaf frame std::string LeafFrame = ContextStrStack.back(); @@ -624,14 +626,15 @@ } FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe( - SmallVectorImpl &ContextStrStack, const PseudoProbe *LeafProbe, - ProfiledBinary *Binary) { + SmallVectorImpl &ContextStrStack, + const MCDecodedPseudoProbe *LeafProbe, ProfiledBinary *Binary) { + // Explicitly copy the context for appending the leaf context SmallVector ContextStrStackCopy(ContextStrStack.begin(), ContextStrStack.end()); Binary->getInlineContextForProbe(LeafProbe, ContextStrStackCopy, true); - const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->GUID); - bool WasLeafInlined = LeafProbe->InlineTree->hasInlineSite(); + const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->getGuid()); + bool WasLeafInlined = LeafProbe->getInlineTreeNode()->hasInlineSite(); return getFunctionProfileForLeafProbe(ContextStrStackCopy, FuncDesc, WasLeafInlined); } diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h --- a/llvm/tools/llvm-profgen/ProfiledBinary.h +++ b/llvm/tools/llvm-profgen/ProfiledBinary.h @@ -10,7 +10,6 @@ #define LLVM_TOOLS_LLVM_PROFGEN_PROFILEDBINARY_H #include "CallContext.h" -#include "PseudoProbe.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/Symbolize/Symbolize.h" @@ -22,6 +21,7 @@ #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCTargetOptions.h" @@ -136,7 +136,7 @@ std::unique_ptr Symbolizer; // Pseudo probe decoder - PseudoProbeDecoder ProbeDecoder; + MCPseudoProbeDecoder ProbeDecoder; bool UsePseudoProbes = false; @@ -265,11 +265,12 @@ std::string getExpandedContextStr(const SmallVectorImpl &Stack, bool &WasLeafInlined) const; - const PseudoProbe *getCallProbeForAddr(uint64_t Address) const { + const MCDecodedPseudoProbe *getCallProbeForAddr(uint64_t Address) const { return ProbeDecoder.getCallProbeForAddr(Address); } + void - getInlineContextForProbe(const PseudoProbe *Probe, + getInlineContextForProbe(const MCDecodedPseudoProbe *Probe, SmallVectorImpl &InlineContextStack, bool IncludeLeaf = false) const { return ProbeDecoder.getInlineContextForProbe(Probe, InlineContextStack, @@ -278,10 +279,12 @@ const AddressProbesMap &getAddress2ProbesMap() const { return ProbeDecoder.getAddress2ProbesMap(); } - const PseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) { + const MCPseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) { return ProbeDecoder.getFuncDescForGUID(GUID); } - const PseudoProbeFuncDesc *getInlinerDescForProbe(const PseudoProbe *Probe) { + + const MCPseudoProbeFuncDesc * + getInlinerDescForProbe(const MCDecodedPseudoProbe *Probe) { return ProbeDecoder.getInlinerDescForProbe(Probe); } diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp --- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp +++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp @@ -177,12 +177,16 @@ if (SectionName == ".pseudo_probe_desc") { StringRef Contents = unwrapOrError(Section.getContents(), FileName); - ProbeDecoder.buildGUID2FuncDescMap( - reinterpret_cast(Contents.data()), Contents.size()); + if (!ProbeDecoder.buildGUID2FuncDescMap( + reinterpret_cast(Contents.data()), + Contents.size())) + exitWithError("Pseudo Probe decoder fail in .pseudo_probe_desc section"); } else if (SectionName == ".pseudo_probe") { StringRef Contents = unwrapOrError(Section.getContents(), FileName); - ProbeDecoder.buildAddress2ProbeMap( - reinterpret_cast(Contents.data()), Contents.size()); + if (!ProbeDecoder.buildAddress2ProbeMap( + reinterpret_cast(Contents.data()), + Contents.size())) + exitWithError("Pseudo Probe decoder fail in .pseudo_probe section"); // set UsePseudoProbes flag, used for PerfReader UsePseudoProbes = true; } diff --git a/llvm/tools/llvm-profgen/PseudoProbe.h b/llvm/tools/llvm-profgen/PseudoProbe.h deleted file mode 100644 --- a/llvm/tools/llvm-profgen/PseudoProbe.h +++ /dev/null @@ -1,227 +0,0 @@ -//===--- PseudoProbe.h - Pseudo probe decoding utilities ---------*- C++-*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#ifndef LLVM_TOOLS_LLVM_PROFGEN_PSEUDOPROBE_H -#define LLVM_TOOLS_LLVM_PROFGEN_PSEUDOPROBE_H - -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Twine.h" -#include "llvm/IR/PseudoProbe.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/SampleProfileProbe.h" -#include -#include -#include -#include -#include -#include -#include - -namespace llvm { -namespace sampleprof { - -enum PseudoProbeAttributes { RESERVED = 1 }; - -// Use func GUID and index as the location info of the inline site -using InlineSite = std::tuple; - -struct PseudoProbe; - -// Tree node to represent the inline relation and its inline site, we use a -// dummy root in the PseudoProbeDecoder to lead the tree, the outlined -// function will directly be the children of the dummy root. For the inlined -// function, all the inlinee will be connected to its inlineer, then further to -// its outlined function. Pseudo probes originating from the function stores the -// tree's leaf node which we can process backwards to get its inline context -class PseudoProbeInlineTree { - std::vector ProbeVector; - - struct InlineSiteHash { - uint64_t operator()(const InlineSite &Site) const { - return std::get<0>(Site) ^ std::get<1>(Site); - } - }; - using InlinedProbeTreeMap = - std::unordered_map, - InlineSiteHash>; - InlinedProbeTreeMap Children; - -public: - // Inlinee function GUID - uint64_t GUID = 0; - // Inline site to indicate the location in its inliner. As the node could also - // be an outlined function, it will use a dummy InlineSite whose GUID and - // Index is 0 connected to the dummy root - InlineSite ISite; - // Used for decoding - uint32_t ChildrenToProcess = 0; - // Caller node of the inline site - PseudoProbeInlineTree *Parent; - - PseudoProbeInlineTree(){}; - PseudoProbeInlineTree(const InlineSite &Site) : ISite(Site){}; - - PseudoProbeInlineTree *getOrAddNode(const InlineSite &Site) { - auto Ret = - Children.emplace(Site, std::make_unique(Site)); - Ret.first->second->Parent = this; - return Ret.first->second.get(); - } - - InlinedProbeTreeMap &getChildren() { return Children; } - std::vector &getProbes() { return ProbeVector; } - void addProbes(PseudoProbe *Probe) { ProbeVector.push_back(Probe); } - // Return false if it's a dummy inline site - bool hasInlineSite() const { return std::get<0>(ISite) != 0; } -}; - -// Function descriptor decoded from .pseudo_probe_desc section -struct PseudoProbeFuncDesc { - uint64_t FuncGUID = 0; - uint64_t FuncHash = 0; - std::string FuncName; - - PseudoProbeFuncDesc(uint64_t GUID, uint64_t Hash, StringRef Name) - : FuncGUID(GUID), FuncHash(Hash), FuncName(Name){}; - - void print(raw_ostream &OS); -}; - -// GUID to PseudoProbeFuncDesc map -using GUIDProbeFunctionMap = std::unordered_map; -// Address to pseudo probes map. -using AddressProbesMap = std::unordered_map>; - -/* -A pseudo probe has the format like below: - INDEX (ULEB128) - TYPE (uint4) - 0 - block probe, 1 - indirect call, 2 - direct call - ATTRIBUTE (uint3) - 1 - reserved - ADDRESS_TYPE (uint1) - 0 - code address, 1 - address delta - CODE_ADDRESS (uint64 or ULEB128) - code address or address delta, depending on Flag -*/ -struct PseudoProbe { - uint64_t Address; - uint64_t GUID; - uint32_t Index; - PseudoProbeType Type; - uint8_t Attribute; - PseudoProbeInlineTree *InlineTree; - const static uint32_t PseudoProbeFirstId = - static_cast(PseudoProbeReservedId::Last) + 1; - - PseudoProbe(uint64_t Ad, uint64_t G, uint32_t I, PseudoProbeType K, - uint8_t At, PseudoProbeInlineTree *Tree) - : Address(Ad), GUID(G), Index(I), Type(K), Attribute(At), - InlineTree(Tree){}; - - bool isEntry() const { return Index == PseudoProbeFirstId; } - bool isBlock() const { return Type == PseudoProbeType::Block; } - bool isIndirectCall() const { return Type == PseudoProbeType::IndirectCall; } - bool isDirectCall() const { return Type == PseudoProbeType::DirectCall; } - bool isCall() const { return isIndirectCall() || isDirectCall(); } - - PseudoProbeInlineTree *getInlineTreeNode() const { return InlineTree; } - - // Get the inlined context by traversing current inline tree backwards, - // each tree node has its InlineSite which is taken as the context. - // \p ContextStack is populated in root to leaf order - void getInlineContext(SmallVectorImpl &ContextStack, - const GUIDProbeFunctionMap &GUID2FuncMAP, - bool ShowName) const; - // Helper function to get the string from context stack - std::string getInlineContextStr(const GUIDProbeFunctionMap &GUID2FuncMAP, - bool ShowName) const; - // Print pseudo probe while disassembling - void print(raw_ostream &OS, const GUIDProbeFunctionMap &GUID2FuncMAP, - bool ShowName); -}; - -/* -Decode pseudo probe info from ELF section, used along with ELF reader -Two sections are decoded here: - 1) \fn buildGUID2FunctionMap is responsible for .pseudo_probe_desc - section which encodes all function descriptors. - 2) \fn buildAddress2ProbeMap is responsible for .pseudoprobe section - which encodes an inline function forest and each tree includes its - inlined function and all pseudo probes inside the function. -see \file MCPseudoProbe.h for the details of the section encoding format. -*/ -class PseudoProbeDecoder { - // GUID to PseudoProbeFuncDesc map. - GUIDProbeFunctionMap GUID2FuncDescMap; - - // Address to probes map. - AddressProbesMap Address2ProbesMap; - - // The dummy root of the inline trie, all the outlined function will directly - // be the children of the dummy root, all the inlined function will be the - // children of its inlineer. So the relation would be like: - // DummyRoot --> OutlinedFunc --> InlinedFunc1 --> InlinedFunc2 - PseudoProbeInlineTree DummyInlineRoot; - - /// Points to the current location in the buffer. - const uint8_t *Data = nullptr; - - /// Points to the end of the buffer. - const uint8_t *End = nullptr; - - /// SectionName used for debug - std::string SectionName; - - // Decoding helper function - template T readUnencodedNumber(); - template T readUnsignedNumber(); - template T readSignedNumber(); - StringRef readString(uint32_t Size); - -public: - // Decode pseudo_probe_desc section to build GUID to PseudoProbeFuncDesc map. - void buildGUID2FuncDescMap(const uint8_t *Start, std::size_t Size); - - // Decode pseudo_probe section to build address to probes map. - void buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size); - - // Print pseudo_probe_desc section info - void printGUID2FuncDescMap(raw_ostream &OS); - - // Print pseudo_probe section info, used along with show-disassembly - void printProbeForAddress(raw_ostream &OS, uint64_t Address); - - // Look up the probe of a call for the input address - const PseudoProbe *getCallProbeForAddr(uint64_t Address) const; - - const PseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) const; - - // Helper function to populate one probe's inline stack into - // \p InlineContextStack. - // Current leaf location info will be added if IncludeLeaf is true - // Example: - // Current probe(bar:3) inlined at foo:2 then inlined at main:1 - // IncludeLeaf = true, Output: [main:1, foo:2, bar:3] - // IncludeLeaf = false, Output: [main:1, foo:2] - void - getInlineContextForProbe(const PseudoProbe *Probe, - SmallVectorImpl &InlineContextStack, - bool IncludeLeaf) const; - - const AddressProbesMap &getAddress2ProbesMap() const { - return Address2ProbesMap; - } - - const PseudoProbeFuncDesc * - getInlinerDescForProbe(const PseudoProbe *Probe) const; -}; - -} // end namespace sampleprof -} // end namespace llvm - -#endif diff --git a/llvm/tools/llvm-profgen/PseudoProbe.cpp b/llvm/tools/llvm-profgen/PseudoProbe.cpp deleted file mode 100644 --- a/llvm/tools/llvm-profgen/PseudoProbe.cpp +++ /dev/null @@ -1,341 +0,0 @@ -//===--- PseudoProbe.cpp - Pseudo probe decoding utilities ------*- C++-*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "PseudoProbe.h" -#include "ErrorHandling.h" -#include "llvm/Support/Endian.h" -#include "llvm/Support/LEB128.h" -#include "llvm/Support/raw_ostream.h" -#include -#include - -using namespace llvm; -using namespace sampleprof; -using namespace support; - -namespace llvm { -namespace sampleprof { - -static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP, - uint64_t GUID) { - auto It = GUID2FuncMAP.find(GUID); - assert(It != GUID2FuncMAP.end() && - "Probe function must exist for a valid GUID"); - return It->second.FuncName; -} - -void PseudoProbeFuncDesc::print(raw_ostream &OS) { - OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n"; - OS << "Hash: " << FuncHash << "\n"; -} - -void PseudoProbe::getInlineContext(SmallVectorImpl &ContextStack, - const GUIDProbeFunctionMap &GUID2FuncMAP, - bool ShowName) const { - uint32_t Begin = ContextStack.size(); - PseudoProbeInlineTree *Cur = InlineTree; - // It will add the string of each node's inline site during iteration. - // Note that it won't include the probe's belonging function(leaf location) - while (Cur->hasInlineSite()) { - std::string ContextStr; - if (ShowName) { - StringRef FuncName = - getProbeFNameForGUID(GUID2FuncMAP, std::get<0>(Cur->ISite)); - ContextStr += FuncName.str(); - } else { - ContextStr += Twine(std::get<0>(Cur->ISite)).str(); - } - ContextStr += ":"; - ContextStr += Twine(std::get<1>(Cur->ISite)).str(); - ContextStack.emplace_back(ContextStr); - Cur = Cur->Parent; - } - // Make the ContextStack in caller-callee order - std::reverse(ContextStack.begin() + Begin, ContextStack.end()); -} - -std::string -PseudoProbe::getInlineContextStr(const GUIDProbeFunctionMap &GUID2FuncMAP, - bool ShowName) const { - std::ostringstream OContextStr; - SmallVector ContextStack; - getInlineContext(ContextStack, GUID2FuncMAP, ShowName); - for (auto &CxtStr : ContextStack) { - if (OContextStr.str().size()) - OContextStr << " @ "; - OContextStr << CxtStr; - } - return OContextStr.str(); -} - -static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall", - "DirectCall"}; - -void PseudoProbe::print(raw_ostream &OS, - const GUIDProbeFunctionMap &GUID2FuncMAP, - bool ShowName) { - OS << "FUNC: "; - if (ShowName) { - StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, GUID); - OS << FuncName.str() << " "; - } else { - OS << GUID << " "; - } - OS << "Index: " << Index << " "; - OS << "Type: " << PseudoProbeTypeStr[static_cast(Type)] << " "; - - std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP, ShowName); - if (InlineContextStr.size()) { - OS << "Inlined: @ "; - OS << InlineContextStr; - } - OS << "\n"; -} - -template T PseudoProbeDecoder::readUnencodedNumber() { - if (Data + sizeof(T) > End) { - exitWithError("Decode unencoded number error in " + SectionName + - " section"); - } - T Val = endian::readNext(Data); - return Val; -} - -template T PseudoProbeDecoder::readUnsignedNumber() { - unsigned NumBytesRead = 0; - uint64_t Val = decodeULEB128(Data, &NumBytesRead); - if (Val > std::numeric_limits::max() || (Data + NumBytesRead > End)) { - exitWithError("Decode number error in " + SectionName + " section"); - } - Data += NumBytesRead; - return static_cast(Val); -} - -template T PseudoProbeDecoder::readSignedNumber() { - unsigned NumBytesRead = 0; - int64_t Val = decodeSLEB128(Data, &NumBytesRead); - if (Val > std::numeric_limits::max() || (Data + NumBytesRead > End)) { - exitWithError("Decode number error in " + SectionName + " section"); - } - Data += NumBytesRead; - return static_cast(Val); -} - -StringRef PseudoProbeDecoder::readString(uint32_t Size) { - StringRef Str(reinterpret_cast(Data), Size); - if (Data + Size > End) { - exitWithError("Decode string error in " + SectionName + " section"); - } - Data += Size; - return Str; -} - -void PseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start, - std::size_t Size) { - // The pseudo_probe_desc section has a format like: - // .section .pseudo_probe_desc,"",@progbits - // .quad -5182264717993193164 // GUID - // .quad 4294967295 // Hash - // .uleb 3 // Name size - // .ascii "foo" // Name - // .quad -2624081020897602054 - // .quad 174696971957 - // .uleb 34 - // .ascii "main" -#ifndef NDEBUG - SectionName = "pseudo_probe_desc"; -#endif - Data = Start; - End = Data + Size; - - while (Data < End) { - uint64_t GUID = readUnencodedNumber(); - uint64_t Hash = readUnencodedNumber(); - uint32_t NameSize = readUnsignedNumber(); - StringRef Name = FunctionSamples::getCanonicalFnName(readString(NameSize)); - - // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap - GUID2FuncDescMap.emplace(GUID, PseudoProbeFuncDesc(GUID, Hash, Name)); - } - assert(Data == End && "Have unprocessed data in pseudo_probe_desc section"); -} - -void PseudoProbeDecoder::buildAddress2ProbeMap(const uint8_t *Start, - std::size_t Size) { - // The pseudo_probe section encodes an inline forest and each tree has a - // format like: - // FUNCTION BODY (one for each uninlined function present in the text - // section) - // GUID (uint64) - // GUID of the function - // NPROBES (ULEB128) - // Number of probes originating from this function. - // NUM_INLINED_FUNCTIONS (ULEB128) - // Number of callees inlined into this function, aka number of - // first-level inlinees - // PROBE RECORDS - // A list of NPROBES entries. Each entry contains: - // INDEX (ULEB128) - // TYPE (uint4) - // 0 - block probe, 1 - indirect call, 2 - direct call - // ATTRIBUTE (uint3) - // 1 - reserved - // ADDRESS_TYPE (uint1) - // 0 - code address, 1 - address delta - // CODE_ADDRESS (uint64 or ULEB128) - // code address or address delta, depending on Flag - // INLINED FUNCTION RECORDS - // A list of NUM_INLINED_FUNCTIONS entries describing each of the - // inlined callees. Each record contains: - // INLINE SITE - // Index of the callsite probe (ULEB128) - // FUNCTION BODY - // A FUNCTION BODY entry describing the inlined function. -#ifndef NDEBUG - SectionName = "pseudo_probe"; -#endif - Data = Start; - End = Data + Size; - - PseudoProbeInlineTree *Root = &DummyInlineRoot; - PseudoProbeInlineTree *Cur = &DummyInlineRoot; - uint64_t LastAddr = 0; - uint32_t Index = 0; - // A DFS-based decoding - while (Data < End) { - if (Root == Cur) { - // Use a sequential id for top level inliner. - Index = Root->getChildren().size(); - } else { - // Read inline site for inlinees - Index = readUnsignedNumber(); - } - // Switch/add to a new tree node(inlinee) - Cur = Cur->getOrAddNode(std::make_tuple(Cur->GUID, Index)); - // Read guid - Cur->GUID = readUnencodedNumber(); - // Read number of probes in the current node. - uint32_t NodeCount = readUnsignedNumber(); - // Read number of direct inlinees - Cur->ChildrenToProcess = readUnsignedNumber(); - // Read all probes in this node - for (std::size_t I = 0; I < NodeCount; I++) { - // Read index - uint32_t Index = readUnsignedNumber(); - // Read type | flag. - uint8_t Value = readUnencodedNumber(); - uint8_t Kind = Value & 0xf; - uint8_t Attr = (Value & 0x70) >> 4; - // Read address - uint64_t Addr = 0; - if (Value & 0x80) { - int64_t Offset = readSignedNumber(); - Addr = LastAddr + Offset; - } else { - Addr = readUnencodedNumber(); - } - // Populate Address2ProbesMap - auto &Probes = Address2ProbesMap[Addr]; - Probes.emplace_back(Addr, Cur->GUID, Index, PseudoProbeType(Kind), Attr, - Cur); - Cur->addProbes(&Probes.back()); - LastAddr = Addr; - } - - // Look for the parent for the next node by subtracting the current - // node count from tree counts along the parent chain. The first node - // in the chain that has a non-zero tree count is the target. - while (Cur != Root) { - if (Cur->ChildrenToProcess == 0) { - Cur = Cur->Parent; - if (Cur != Root) { - assert(Cur->ChildrenToProcess > 0 && - "Should have some unprocessed nodes"); - Cur->ChildrenToProcess -= 1; - } - } else { - break; - } - } - } - - assert(Data == End && "Have unprocessed data in pseudo_probe section"); - assert(Cur == Root && - " Cur should point to root when the forest is fully built up"); -} - -void PseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) { - OS << "Pseudo Probe Desc:\n"; - // Make the output deterministic - std::map OrderedMap(GUID2FuncDescMap.begin(), - GUID2FuncDescMap.end()); - for (auto &I : OrderedMap) { - I.second.print(OS); - } -} - -void PseudoProbeDecoder::printProbeForAddress(raw_ostream &OS, - uint64_t Address) { - auto It = Address2ProbesMap.find(Address); - if (It != Address2ProbesMap.end()) { - for (auto &Probe : It->second) { - OS << " [Probe]:\t"; - Probe.print(OS, GUID2FuncDescMap, true); - } - } -} - -const PseudoProbe * -PseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const { - auto It = Address2ProbesMap.find(Address); - if (It == Address2ProbesMap.end()) - return nullptr; - const auto &Probes = It->second; - - const PseudoProbe *CallProbe = nullptr; - for (const auto &Probe : Probes) { - if (Probe.isCall()) { - assert(!CallProbe && - "There should be only one call probe corresponding to address " - "which is a callsite."); - CallProbe = &Probe; - } - } - return CallProbe; -} - -const PseudoProbeFuncDesc * -PseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const { - auto It = GUID2FuncDescMap.find(GUID); - assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist"); - return &It->second; -} - -void PseudoProbeDecoder::getInlineContextForProbe( - const PseudoProbe *Probe, SmallVectorImpl &InlineContextStack, - bool IncludeLeaf) const { - Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap, true); - if (!IncludeLeaf) - return; - // Note that the context from probe doesn't include leaf frame, - // hence we need to retrieve and prepend leaf if requested. - const auto *FuncDesc = getFuncDescForGUID(Probe->GUID); - InlineContextStack.emplace_back(FuncDesc->FuncName + ":" + - Twine(Probe->Index).str()); -} - -const PseudoProbeFuncDesc * -PseudoProbeDecoder::getInlinerDescForProbe(const PseudoProbe *Probe) const { - PseudoProbeInlineTree *InlinerNode = Probe->InlineTree; - if (!InlinerNode->hasInlineSite()) - return nullptr; - return getFuncDescForGUID(std::get<0>(InlinerNode->ISite)); -} - -} // end namespace sampleprof -} // end namespace llvm