diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -66,6 +66,7 @@ class MCTargetOptions; class MDNode; class Module; +class PseudoProbeHandler; class raw_ostream; class StackMaps; class StringRef; @@ -206,6 +207,10 @@ /// If the target supports dwarf debug info, this pointer is non-null. DwarfDebug *DD = nullptr; + /// A handler that supports pseudo probe emission with embedded inline + /// context. + PseudoProbeHandler *PP = nullptr; + /// If the current module uses dwarf CFI annotations strictly for debugging. bool isCFIMoveForDebugging = false; @@ -360,6 +365,8 @@ void emitBBAddrMapSection(const MachineFunction &MF); + void emitPseudoProbe(const MachineInstr &MI); + void emitRemarksSection(remarks::RemarkStreamer &RS); enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -76,6 +76,9 @@ /// Return metadata containing the section prefix for a function. MDNode *createFunctionSectionPrefix(StringRef Prefix); + /// Return metadata containing the pseudo probe descriptor for a function. + MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, Function *F); + //===------------------------------------------------------------------===// // Range metadata. //===------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h --- a/llvm/include/llvm/IR/PseudoProbe.h +++ b/llvm/include/llvm/IR/PseudoProbe.h @@ -15,6 +15,8 @@ namespace llvm { +constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc"; + enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall }; struct PseudoProbeDwarfDiscriminator { diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h --- a/llvm/include/llvm/MC/MCAssembler.h +++ b/llvm/include/llvm/MC/MCAssembler.h @@ -202,6 +202,7 @@ bool relaxCVInlineLineTable(MCAsmLayout &Layout, MCCVInlineLineTableFragment &DF); bool relaxCVDefRange(MCAsmLayout &Layout, MCCVDefRangeFragment &DF); + bool relaxPseudoProbeAddr(MCAsmLayout &Layout, MCPseudoProbeAddrFragment &DF); /// finishLayout - Finalize a layout, including fragment lowering. void finishLayout(MCAsmLayout &Layout); diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -22,6 +22,7 @@ #include "llvm/BinaryFormat/XCOFF.h" #include "llvm/MC/MCAsmMacro.h" #include "llvm/MC/MCDwarf.h" +#include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/SectionKind.h" @@ -199,6 +200,9 @@ /// The Compile Unit ID that we are currently processing. unsigned DwarfCompileUnitID = 0; + /// A collection of MCPseudoProbe in the current module + MCPseudoProbeTable PseudoProbeTable; + // Sections are differentiated by the quadruple (section_name, group_name, // unique_id, link_to_symbol_name). Sections sharing the same quadruple are // combined into one section. @@ -749,6 +753,8 @@ } void undefineMacro(StringRef Name) { MacroMap.erase(Name); } + + MCPseudoProbeTable &getMCPseudoProbeTable() { return PseudoProbeTable; } }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h --- a/llvm/include/llvm/MC/MCFragment.h +++ b/llvm/include/llvm/MC/MCFragment.h @@ -47,6 +47,7 @@ FT_SymbolId, FT_CVInlineLines, FT_CVDefRange, + FT_PseudoProbe, FT_Dummy }; @@ -147,6 +148,7 @@ case MCFragment::FT_Data: case MCFragment::FT_Dwarf: case MCFragment::FT_DwarfFrame: + case MCFragment::FT_PseudoProbe: return true; } } @@ -591,6 +593,23 @@ return F->getKind() == MCFragment::FT_BoundaryAlign; } }; + +class MCPseudoProbeAddrFragment : public MCEncodedFragmentWithFixups<8, 1> { + /// The expression for the difference of the two symbols that + /// make up the address delta between two .pseudoprobe directives. + const MCExpr *AddrDelta; + +public: + MCPseudoProbeAddrFragment(const MCExpr *AddrDelta, MCSection *Sec = nullptr) + : MCEncodedFragmentWithFixups<8, 1>(FT_PseudoProbe, false, Sec), + AddrDelta(AddrDelta) {} + + const MCExpr &getAddrDelta() const { return *AddrDelta; } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_PseudoProbe; + } +}; } // end namespace llvm #endif // LLVM_MC_MCFRAGMENT_H diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -174,6 +174,10 @@ /// Section containing metadata on function stack sizes. MCSection *StackSizesSection = nullptr; + /// Section for pseudo probe information used by AutoFDO + MCSection *PseudoProbeSection = nullptr; + MCSection *PseudoProbeDescSection = nullptr; + // ELF specific sections. MCSection *DataRelROSection = nullptr; MCSection *MergeableConst4Section = nullptr; @@ -340,6 +344,10 @@ MCSection *getBBAddrMapSection(const MCSection &TextSec) const; + MCSection *getPseudoProbeSection(const MCSection *TextSec) const; + + MCSection *getPseudoProbeDescSection(StringRef FuncName) const; + // ELF specific sections. MCSection *getDataRelROSection() const { return DataRelROSection; } const MCSection *getMergeableConst4Section() const { diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/MC/MCPseudoProbe.h @@ -0,0 +1,176 @@ +//===- MCPseudoProbe.h - Pseudo probe encoding support ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the declaration of the MCPseudoProbe to support the pseudo +// probe encoding for AutoFDO. Pseudo probes together with their inline context +// are encoded in a DFS recursive way in the .pseudoprobe sections. For each +// .pseudoprobe section, the encoded binary data consist of a single or mutiple +// function records each for one outlined function. A function record has the +// following format : +// +// FUNCTION BODY (one for each outlined function present in the text section) +// GUID (uint64) +// GUID of the function +// NPROBES (ULEB128) +// Number of probes originating from this function. +// NUM_INLINED_FUNCTIONS (ULEB128) +// Number of callees inlined into this function, aka number of +// first-level inlinees +// PROBE RECORDS +// A list of NPROBES entries. Each entry contains: +// INDEX (ULEB128) +// TYPE (uint4) +// 0 - block probe, 1 - indirect call, 2 - direct call +// ATTRIBUTE (uint3) +// reserved +// ADDRESS_TYPE (uint1) +// 0 - code address, 1 - address delta +// CODE_ADDRESS (uint64 or ULEB128) +// code address or address delta, depending on ADDRESS_TYPE +// INLINED FUNCTION RECORDS +// A list of NUM_INLINED_FUNCTIONS entries describing each of the inlined +// callees. Each record contains: +// INLINE SITE +// GUID of the inlinee (uint64) +// ID of the callsite probe (ULEB128) +// FUNCTION BODY +// A FUNCTION BODY entry describing the inlined function. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCPSEUDOPROBE_H +#define LLVM_MC_MCPSEUDOPROBE_H + +#include "llvm/ADT/MapVector.h" +#include "llvm/MC/MCSection.h" +#include +#include +#include + +namespace llvm { + +class MCStreamer; +class MCSymbol; +class MCObjectStreamer; + +enum class MCPseudoProbeFlag { + // If set, indicates that the probe is encoded as an address delta + // instead of a real code address. + AddressDelta = 0x1, +}; + +/// Instances of this class represent a pseudo probe instance for a pseudo probe +/// table entry, which is created during a machine instruction is assembled and +/// uses an address from a temporary label created at the current address in the +/// current section. +class MCPseudoProbe { + MCSymbol *Label; + uint64_t Guid; + uint64_t Index; + uint8_t Type; + uint8_t Attributes; + +public: + MCPseudoProbe(MCSymbol *Label, uint64_t Guid, uint64_t Index, uint64_t Type, + uint64_t Attributes) + : Label(Label), Guid(Guid), Index(Index), Type(Type), + Attributes(Attributes) { + assert(Type < 0xFF && Attributes < 0xFF); + } + + MCSymbol *getLabel() const { return Label; } + + uint64_t getGuid() const { return Guid; } + + uint64_t getIndex() const { return Index; } + + uint8_t getType() const { return Type; } + + uint8_t getAttributes() const { return Attributes; } + + void Emit(MCObjectStreamer *MCOS, const MCPseudoProbe *LastProbe) const; +}; + +// An inline frame has the form +using InlineSite = std::tuple; +using MCPseudoProbeInlineStack = SmallVector; + +// A Tri-tree based data structure to group probes by inline stack. +// A tree is allocated for a standalone .text section. A fake +// instance is created as the root of a tree. +// A real instance of this class is created for each function, either an +// unlined function that has code in .text section or an inlined function. +class MCPseudoProbeInlineTree { + uint64_t Guid; + // Set of probes that come with the function. + std::vector Probes; + // Use std::map for a deterministic output. + std::map Inlinees; + + // Root node has a GUID 0. + bool isRoot() { return Guid == 0; } + MCPseudoProbeInlineTree *getOrAddNode(InlineSite Site); + +public: + MCPseudoProbeInlineTree() = default; + MCPseudoProbeInlineTree(uint64_t Guid) : Guid(Guid) {} + ~MCPseudoProbeInlineTree(); + void addPseudoProbe(const MCPseudoProbe &Probe, + const MCPseudoProbeInlineStack &inlineStack); + void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *&LastProbe); +}; + +/// Instances of this class represent the pseudo probes inserted into a compile +/// unit. +class MCPseudoProbeSection { +public: + void addPseudoProbe(MCSection *Sec, const MCPseudoProbe &Probe, + const MCPseudoProbeInlineStack &InlineStack) { + MCProbeDivisions[Sec].addPseudoProbe(Probe, InlineStack); + } + + // TODO: Sort by getOrdinal to ensure a determinstic section order + using MCProbeDivisionMap = std::map; + +private: + // A collection of MCPseudoProbe for each text section. The MCPseudoProbes + // are grouped by GUID of the functions where they are from and will be + // encoded by groups. In the comdat scenario where a text section really only + // contains the code of a function solely, the probes associated with a comdat + // function are still grouped by GUIDs due to inlining that can bring probes + // from different functions into one function. + MCProbeDivisionMap MCProbeDivisions; + +public: + const MCProbeDivisionMap &getMCProbes() const { return MCProbeDivisions; } + + bool empty() const { return MCProbeDivisions.empty(); } + + void Emit(MCObjectStreamer *MCOS); +}; + +class MCPseudoProbeTable { + // A collection of MCPseudoProbe in the current module grouped by text + // sections. MCPseudoProbes will be encoded into a corresponding + // .pseudoprobe section. With functions emitted as separate comdats, + // a text section really only contains the code of a function solely, and the + // probes associated with the text section will be emitted into a standalone + // .pseudoprobe section that shares the same comdat group with the function. + MCPseudoProbeSection MCProbeSections; + +public: + static void Emit(MCObjectStreamer *MCOS); + + MCPseudoProbeSection &getProbeSections() { return MCProbeSections; } + +#ifndef NDEBUG + static int DdgPrintIndent; +#endif +}; +} // end namespace llvm + +#endif // LLVM_MC_MCPSEUDOPROBE_H diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -21,6 +21,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCLinkerOptimizationHint.h" +#include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWinEH.h" #include "llvm/Support/Error.h" @@ -1046,6 +1047,11 @@ /// Emit the given \p Instruction into the current section. virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI); + /// Emit the a pseudo probe into the current section. + virtual void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, + uint64_t Attr, + const MCPseudoProbeInlineStack &InlineStack); + /// Set the bundle alignment mode from now on in the section. /// The argument is the power of 2 to which the alignment is set. The /// value 0 means turn the bundle alignment off. diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h --- a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -37,18 +37,27 @@ class SampleProfileProber { public: // Give an empty module id when the prober is not used for instrumentation. - SampleProfileProber(Function &F); + SampleProfileProber(Function &F, const std::string &CurModuleUniqueId); void instrumentOneFunc(Function &F, TargetMachine *TM); private: Function *getFunction() const { return F; } + uint64_t getFunctionHash() const { return FunctionHash; } uint32_t getBlockId(const BasicBlock *BB) const; uint32_t getCallsiteId(const Instruction *call) const; + void computeCFGHash(); void computeProbeIdForBlocks(); void computeProbeIdForCallsites(); Function *F; + /// The current module ID that is used to name a static object as a comdat + /// group. + std::string CurModuleUniqueId; + + /// A CFG hash code used to identify a function code changes. + uint64_t FunctionHash; + /// Map basic blocks to the their pseudo probe ids. BlockIdMap BlockProbeIds; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -14,6 +14,7 @@ #include "CodeViewDebug.h" #include "DwarfDebug.h" #include "DwarfException.h" +#include "PseudoProbePrinter.h" #include "WasmException.h" #include "WinCFGuard.h" #include "WinException.h" @@ -77,6 +78,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/PseudoProbe.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/MC/MCAsmInfo.h" @@ -333,6 +335,9 @@ } } + if (M.getNamedMetadata(PseudoProbeDescMetadataName)) + PP = new PseudoProbeHandler(this, &M); + switch (MAI->getExceptionHandlingType()) { case ExceptionHandling::SjLj: case ExceptionHandling::DwarfCFI: @@ -1083,6 +1088,15 @@ OutStreamer->PopSection(); } +void AsmPrinter::emitPseudoProbe(const MachineInstr &MI) { + auto GUID = MI.getOperand(0).getImm(); + auto Index = MI.getOperand(1).getImm(); + auto Type = MI.getOperand(2).getImm(); + auto Attr = MI.getOperand(3).getImm(); + DILocation *DebugLoc = MI.getDebugLoc(); + PP->emitPseudoProbe(GUID, Index, Type, Attr, DebugLoc); +} + void AsmPrinter::emitStackSizeSection(const MachineFunction &MF) { if (!MF.getTarget().Options.EmitStackSizeSection) return; @@ -1216,6 +1230,9 @@ case TargetOpcode::KILL: if (isVerbose()) emitKill(&MI, *this); break; + case TargetOpcode::PSEUDO_PROBE: + emitPseudoProbe(MI); + break; default: emitInstruction(&MI); if (CanDoExtraAnalysis) { diff --git a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt --- a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt +++ b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt @@ -20,6 +20,7 @@ EHStreamer.cpp ErlangGCPrinter.cpp OcamlGCPrinter.cpp + PseudoProbePrinter.cpp WinCFGuard.cpp WinException.cpp CodeViewDebug.cpp diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h @@ -0,0 +1,41 @@ +//===- PseudoProbePrinter.h - Pseudo probe encoding support -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for writing pseudo probe info into asm files. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_PSEUDOPROBEPRINTER_H +#define LLVM_LIB_CODEGEN_ASMPRINTER_PSEUDOPROBEPRINTER_H + +#include "llvm/ADT/DenseMap.h" + +namespace llvm { + +class AsmPrinter; +class MCStreamer; +class Module; +class DILocation; + +class PseudoProbeHandler { + // Target of pseudo probe emission. + AsmPrinter *Asm; + // Name to GUID map + DenseMap Names; + +public: + PseudoProbeHandler(AsmPrinter *A, Module *M); + void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, + uint64_t Attr, const DILocation *DebugLoc); +#ifndef NDEBUG + void dump() const; +#endif +}; + +} // namespace llvm +#endif // LLVM_LIB_CODEGEN_ASMPRINTER_PSEUDOPROBEPRINTER_H diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp @@ -0,0 +1,80 @@ +//===- llvm/CodeGen/PseudoProbePrinter.cpp - Pseudo Probe Emission -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for writing pseudo probe info into asm files. +// +//===----------------------------------------------------------------------===// + +#include "PseudoProbePrinter.h" +#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PseudoProbe.h" +#include "llvm/MC/MCPseudoProbe.h" +#include "llvm/MC/MCStreamer.h" + +using namespace llvm; + +#define DEBUG_TYPE "pseudoprobe" + +PseudoProbeHandler::PseudoProbeHandler(AsmPrinter *A, Module *M) : Asm(A) { + NamedMDNode *FuncInfo = M->getNamedMetadata(PseudoProbeDescMetadataName); + assert(FuncInfo); + for (const auto *Operand : FuncInfo->operands()) { + const auto *MD = cast(Operand); + auto GUID = + mdconst::dyn_extract(MD->getOperand(0))->getZExtValue(); + auto Name = cast(MD->getOperand(2))->getString(); + // We may see pairs with same name but different GUIDs here in LTO mode, due + // to static same-named functions inlined from other modules into this + // module. Function profiles with the same name will be merged no matter + // whether they are collected on the same function. Therefore we just pick + // up the last pair here to represent the same-named function + // collection and all probes from the collection will be merged into a + // single profile eventually. + Names[Name] = GUID; + } + + LLVM_DEBUG(dump()); +} + +void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index, + uint64_t Type, uint64_t Attr, + const DILocation *DebugLoc) { + // Gather all the inlined-at nodes. + SmallVector InlineStack; + // When it's done InlineStack look like ([66, B], [88, A]) + // which means, Function A inlines function B at line 88, and B inlines C at + // line 66 where C is represented by Guid. + auto InlinedAt = DebugLoc ? DebugLoc->getInlinedAt() : nullptr; + while (InlinedAt) { + const DISubprogram *SP = InlinedAt->getScope()->getSubprogram(); + // Use linkage name for C++ if possible. + auto Name = SP->getLinkageName(); + if (Name.empty()) + Name = SP->getName(); + assert(Names.count(Name)); + uint64_t CallerGuid = Names[Name]; + uint64_t CallerProbeId = PseudoProbeDwarfDiscriminator::extractProbeIndex( + InlinedAt->getDiscriminator()); + InlineStack.push_back({CallerGuid, CallerProbeId}); + InlinedAt = InlinedAt->getInlinedAt(); + } + + Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, InlineStack); +} + +#ifndef NDEBUG +void PseudoProbeHandler::dump() const { + dbgs() << "\n=============================\n"; + dbgs() << "\nFunction Name to GUID map:\n"; + dbgs() << "\n=============================\n"; + for (const auto &item : Names) + dbgs() << "Func: " << item.first << " GUID: " << item.second << "\n"; +} +#endif diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -40,6 +40,7 @@ #include "llvm/IR/Mangler.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PseudoProbe.h" #include "llvm/IR/Type.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" @@ -311,6 +312,29 @@ } } + if (NamedMDNode *FuncInfo = M.getNamedMetadata(PseudoProbeDescMetadataName)) { + // Emit a descriptor for every function including functions that have an + // available external linkage. We may not want this for imported functions + // that has code in another thinLTO module but we don't have a good way to + // tell them apart from inline functions defined in header files. Therefore + // we put each descriptor in a separate comdat section and rely on the + // linker to deduplicate. + for (const auto *Operand : FuncInfo->operands()) { + const auto *MD = cast(Operand); + auto GUID = mdconst::dyn_extract(MD->getOperand(0)); + auto Hash = mdconst::dyn_extract(MD->getOperand(1)); + auto Name = cast(MD->getOperand(2)); + auto *S = C.getObjectFileInfo()->getPseudoProbeDescSection( + TM->getFunctionSections() ? Name->getString() : StringRef()); + + Streamer.SwitchSection(S); + Streamer.emitInt64(GUID->getZExtValue()); + Streamer.emitInt64(Hash->getZExtValue()); + Streamer.emitULEB128IntValue(Name->getString().size()); + Streamer.emitBytes(Name->getString()); + } + } + unsigned Version = 0; unsigned Flags = 0; StringRef Section; diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -305,3 +305,13 @@ }; return MDNode::get(Context, Vals); } + +MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, + Function *F) { + auto *Int64Ty = Type::getInt64Ty(Context); + SmallVector Ops(3); + Ops[0] = createConstant(ConstantInt::get(Int64Ty, GUID)); + Ops[1] = createConstant(ConstantInt::get(Int64Ty, Hash)); + Ops[2] = createString(F->getName()); + return MDNode::get(Context, Ops); +} diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt --- a/llvm/lib/MC/CMakeLists.txt +++ b/llvm/lib/MC/CMakeLists.txt @@ -32,6 +32,7 @@ MCObjectFileInfo.cpp MCObjectStreamer.cpp MCObjectWriter.cpp + MCPseudoProbe.cpp MCRegisterInfo.cpp MCSchedule.cpp MCSection.cpp diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -24,6 +24,7 @@ #include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCRegister.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSectionMachO.h" @@ -350,6 +351,10 @@ void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override; + void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, + uint64_t Attr, + const MCPseudoProbeInlineStack &InlineStack) override; + void emitBundleAlignMode(unsigned AlignPow2) override; void emitBundleLock(bool AlignToEnd) override; void emitBundleUnlock() override; @@ -2122,6 +2127,21 @@ EmitEOL(); } +void MCAsmStreamer::emitPseudoProbe( + uint64_t Guid, uint64_t Index, uint64_t Type, uint64_t Attr, + const MCPseudoProbeInlineStack &InlineStack) { + OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " " + << Attr; + // Emit inline stack like + // @ GUIDDirectCaller:11 @ GUIDCaller:1 @ GUIDmain:3 + if (!InlineStack.empty()) { + for (auto &Site : InlineStack) { + OS << " @ " << std::get<0>(Site) << ":" << std::get<1>(Site); + } + } + EmitEOL(); +} + void MCAsmStreamer::emitBundleAlignMode(unsigned AlignPow2) { OS << "\t.bundle_align_mode " << AlignPow2; EmitEOL(); diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp --- a/llvm/lib/MC/MCAssembler.cpp +++ b/llvm/lib/MC/MCAssembler.cpp @@ -383,6 +383,8 @@ return cast(F).getContents().size(); case MCFragment::FT_CVDefRange: return cast(F).getContents().size(); + case MCFragment::FT_PseudoProbe: + return cast(F).getContents().size(); case MCFragment::FT_Dummy: llvm_unreachable("Should not have been added"); } @@ -704,6 +706,11 @@ OS << DRF.getContents(); break; } + case MCFragment::FT_PseudoProbe: { + const MCPseudoProbeAddrFragment &PF = cast(F); + OS << PF.getContents(); + break; + } case MCFragment::FT_Dummy: llvm_unreachable("Should not have been added"); } @@ -915,6 +922,12 @@ Contents = DF.getContents(); break; } + case MCFragment::FT_PseudoProbe: { + MCPseudoProbeAddrFragment &PF = cast(Frag); + Fixups = PF.getFixups(); + Contents = PF.getContents(); + break; + } } for (const MCFixup &Fixup : Fixups) { uint64_t FixedValue; @@ -1170,6 +1183,26 @@ return OldSize != F.getContents().size(); } +bool MCAssembler::relaxPseudoProbeAddr(MCAsmLayout &Layout, + MCPseudoProbeAddrFragment &PF) { + uint64_t OldSize = PF.getContents().size(); + int64_t AddrDelta; + bool Abs = PF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, Layout); + assert(Abs && "We created a pseudo probe with an invalid expression"); + (void)Abs; + SmallVectorImpl &Data = PF.getContents(); + Data.clear(); + raw_svector_ostream OSE(Data); + PF.getFixups().clear(); + + // Relocations should not be needed in general except on RISC-V which we are + // not targeting for now. + assert(!getBackend().requiresDiffExpressionRelocations()); + // AddrDelta is a signed integer + encodeSLEB128(AddrDelta, OSE, OldSize); + return OldSize != Data.size(); +} + bool MCAssembler::relaxFragment(MCAsmLayout &Layout, MCFragment &F) { switch(F.getKind()) { default: @@ -1191,6 +1224,8 @@ return relaxCVInlineLineTable(Layout, cast(F)); case MCFragment::FT_CVDefRange: return relaxCVDefRange(Layout, cast(F)); + case MCFragment::FT_PseudoProbe: + return relaxPseudoProbeAddr(Layout, cast(F)); } } diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp --- a/llvm/lib/MC/MCFragment.cpp +++ b/llvm/lib/MC/MCFragment.cpp @@ -309,6 +309,9 @@ case FT_CVDefRange: delete cast(this); return; + case FT_PseudoProbe: + delete cast(this); + return; case FT_Dummy: delete cast(this); return; @@ -351,6 +354,9 @@ case MCFragment::FT_SymbolId: OS << "MCSymbolIdFragment"; break; case MCFragment::FT_CVInlineLines: OS << "MCCVInlineLineTableFragment"; break; case MCFragment::FT_CVDefRange: OS << "MCCVDefRangeTableFragment"; break; + case MCFragment::FT_PseudoProbe: + OS << "MCPseudoProbe"; + break; case MCFragment::FT_Dummy: OS << "MCDummyFragment"; break; } @@ -484,6 +490,12 @@ } break; } + case MCFragment::FT_PseudoProbe: { + const auto *OF = cast(this); + OS << "\n "; + OS << " AddrDelta:" << OF->getAddrDelta(); + break; + } case MCFragment::FT_Dummy: break; } diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -497,6 +497,10 @@ Ctx->getELFSection(".eh_frame", EHSectionType, EHSectionFlags); StackSizesSection = Ctx->getELFSection(".stack_sizes", ELF::SHT_PROGBITS, 0); + + PseudoProbeSection = Ctx->getELFSection(".pseudo_probe", DebugSecType, 0); + PseudoProbeDescSection = + Ctx->getELFSection(".pseudo_probe_desc", DebugSecType, 0); } void MCObjectFileInfo::initCOFFMCObjectFileInfo(const Triple &T) { @@ -1018,3 +1022,41 @@ Flags, 0, GroupName, MCSection::NonUniqueID, cast(TextSec.getBeginSymbol())); } + +MCSection * +MCObjectFileInfo::getPseudoProbeSection(const MCSection *TextSec) const { + if (Env == IsELF) { + auto ElfSec = static_cast(TextSec); + // Create a separate section for probes that comes with a comdat function. + if (const MCSymbol *Group = ElfSec->getGroup()) { + auto S = static_cast(PseudoProbeSection); + auto Flags = S->getFlags() | ELF::SHF_GROUP; + return Ctx->getELFSection(S->getName(), S->getType(), Flags, + S->getEntrySize(), Group->getName()); + } + } + return PseudoProbeSection; +} + +MCSection * +MCObjectFileInfo::getPseudoProbeDescSection(StringRef FuncName) const { + if (Env == IsELF) { + // Create a separate comdat group for each function's descriptor in order + // for the linker to deduplicate. The duplication, must be from different + // tranlation unit, can come from: + // 1. Inline functions defined in header files; + // 2. ThinLTO imported funcions; + // 3. Weak-linkage definitions. + // Use a concatenation of the section name and the function name as the + // group name so that descriptor-only groups won't be folded with groups of + // code. + if (TT.supportsCOMDAT() && !FuncName.empty()) { + auto S = static_cast(PseudoProbeDescSection); + auto Flags = S->getFlags() | ELF::SHF_GROUP; + return Ctx->getELFSection(S->getName(), S->getType(), Flags, + S->getEntrySize(), + S->getName() + "_" + FuncName); + } + } + return PseudoProbeDescSection; +} diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -852,6 +852,9 @@ // Dump out the dwarf file & directory tables and line tables. MCDwarfLineTable::Emit(this, getAssembler().getDWARFLinetableParams()); + // Emit pseudo probes for the current module. + MCPseudoProbeTable::Emit(this); + // Update any remaining pending labels with empty data fragments. flushPendingLabels(); diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp --- a/llvm/lib/MC/MCParser/AsmParser.cpp +++ b/llvm/lib/MC/MCParser/AsmParser.cpp @@ -515,6 +515,7 @@ DK_PRINT, DK_ADDRSIG, DK_ADDRSIG_SYM, + DK_PSEUDO_PROBE, DK_END }; @@ -678,6 +679,9 @@ // .print bool parseDirectivePrint(SMLoc DirectiveLoc); + // .pseudoprobe + bool parseDirectivePseudoProbe(); + // Directives to support address-significance tables. bool parseDirectiveAddrsig(); bool parseDirectiveAddrsigSym(); @@ -2202,6 +2206,8 @@ return parseDirectiveAddrsig(); case DK_ADDRSIG_SYM: return parseDirectiveAddrsigSym(); + case DK_PSEUDO_PROBE: + return parseDirectivePseudoProbe(); } return Error(IDLoc, "unknown directive"); @@ -5515,6 +5521,7 @@ DirectiveKindMap[".print"] = DK_PRINT; DirectiveKindMap[".addrsig"] = DK_ADDRSIG; DirectiveKindMap[".addrsig_sym"] = DK_ADDRSIG_SYM; + DirectiveKindMap[".pseudoprobe"] = DK_PSEUDO_PROBE; } MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) { @@ -5770,6 +5777,69 @@ return false; } +bool AsmParser::parseDirectivePseudoProbe() { + int64_t Guid; + int64_t Index; + int64_t Type; + int64_t Attr; + + if (getLexer().is(AsmToken::Integer)) { + if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive")) + return true; + } + + if (getLexer().is(AsmToken::Integer)) { + if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive")) + return true; + } + + if (getLexer().is(AsmToken::Integer)) { + if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive")) + return true; + } + + if (getLexer().is(AsmToken::Integer)) { + if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive")) + return true; + } + + // Parse inline stack like @ GUID:11:12 @ GUID:1:11 @ GUID:3:21 + MCPseudoProbeInlineStack InlineStack; + + while (getLexer().is(AsmToken::At)) { + // eat @ + Lex(); + + int64_t CallerGuid = 0; + if (getLexer().is(AsmToken::Integer)) { + if (parseIntToken(CallerGuid, + "unexpected token in '.pseudoprobe' directive")) + return true; + } + + // eat colon + if (getLexer().is(AsmToken::Colon)) + Lex(); + + int64_t CallerProbeId = 0; + if (getLexer().is(AsmToken::Integer)) { + if (parseIntToken(CallerProbeId, + "unexpected token in '.pseudoprobe' directive")) + return true; + } + + InlineSite Site = {CallerGuid, CallerProbeId}; + InlineStack.push_back(Site); + } + + if (parseToken(AsmToken::EndOfStatement, + "unexpected token in '.pseudoprobe' directive")) + return true; + + getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, InlineStack); + return false; +} + // We are comparing pointers, but the pointers are relative to a single string. // Thus, this should always be deterministic. static int rewritesSort(const AsmRewrite *AsmRewriteA, diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/MC/MCPseudoProbe.cpp @@ -0,0 +1,210 @@ +//===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/MC/MCPseudoProbe.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCStreamer.h" + +#define DEBUG_TYPE "mcpseudoprobe" + +using namespace llvm; + +#ifndef NDEBUG +int MCPseudoProbeTable::DdgPrintIndent = 0; +#endif + +static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A, + const MCSymbol *B) { + MCContext &Context = MCOS->getContext(); + MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None; + const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context); + const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context); + const MCExpr *AddrDelta = + MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context); + return AddrDelta; +} + +void MCPseudoProbe::Emit(MCObjectStreamer *MCOS, + const MCPseudoProbe *LastProbe) const { + // Emit Index + MCOS->emitULEB128IntValue(Index); + // Emit Type and the flag: + // Type (0 - block probe, 1 - indirect call), with bit 5 to 7 for attributes. + // Flag (0 - code address, 128 - address delta). This indicates whether the + // following field is a symbolic code address or an address delta. + assert(Type <= 0xF && Attributes <= 0x7); + uint8_t PackedType = Type | (Attributes << 4); + uint8_t Flag = LastProbe ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0; + MCOS->emitInt8(Flag | PackedType); + + if (LastProbe) { + // Emit the delta between the address label and LastProbe. + const MCExpr *AddrDelta = + buildSymbolDiff(MCOS, Label, LastProbe->getLabel()); + int64_t Delta; + if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) { + MCOS->emitSLEB128IntValue(Delta); + } else { + MCOS->insert(new MCPseudoProbeAddrFragment(AddrDelta)); + } + } else { + // Emit label as a symbolic code address. + MCOS->emitSymbolValue( + Label, MCOS->getContext().getAsmInfo()->getCodePointerSize()); + } + + LLVM_DEBUG({ + dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); + dbgs() << "Probe: " << Index << "\n"; + }); +} + +MCPseudoProbeInlineTree::~MCPseudoProbeInlineTree() { + for (auto &Inlinee : Inlinees) + delete Inlinee.second; +} + +MCPseudoProbeInlineTree * +MCPseudoProbeInlineTree::getOrAddNode(InlineSite Site) { + auto Iter = Inlinees.find(Site); + if (Iter == Inlinees.end()) { + auto Node = new MCPseudoProbeInlineTree(std::get<0>(Site)); + Inlinees[Site] = Node; + return Node; + } else { + return Iter->second; + } +} + +void MCPseudoProbeInlineTree::addPseudoProbe( + const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) { + // The function should not be called on the root. + assert(isRoot()); + + // When it comes here, the input look like: + // Probe: GUID of C, ... + // InlineStack: [66, B], [88, A] + // which means, Function A inlines function B at line 88, and B inlines C at + // line 66. The tri-tree expects a tree path like {[0, A], [88, B], [66, C]} + // to locate the tree node where the probe should be added. Note that the edge + // [0, A] means A is the top-level function we are emitting probes for. + + // Make a [0, A] edge. + // An empty inline stack means the function that the probe originates from + // is a top-level function. + InlineSite Top; + if (InlineStack.empty()) { + Top = InlineSite(Probe.getGuid(), 0); + } else { + Top = InlineSite(std::get<0>(InlineStack.back()), 0); + } + + auto Cur = getOrAddNode(Top); + + // Make interior edges by reverse walking the inline stack. Once it's done, + // Cur should point to the node that the probe originates from. + if (!InlineStack.empty()) { + auto Iter = InlineStack.rbegin(); + auto Index = std::get<1>(*Iter); + Iter++; + for (; Iter != InlineStack.rend(); Iter++) { + // Make an edge by using the previous line number and current GUID. + Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index)); + Index = std::get<1>(*Iter); + } + Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index)); + } + + Cur->Probes.push_back(Probe); +} + +void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS, + const MCPseudoProbe *&LastProbe) { + LLVM_DEBUG({ + dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); + dbgs() << "Group [\n"; + MCPseudoProbeTable::DdgPrintIndent += 2; + }); + // Emit probes grouped by GUID. + if (Guid != 0) { + LLVM_DEBUG({ + dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); + dbgs() << "GUID: " << Guid << "\n"; + }); + // Emit Guid + MCOS->emitInt64(Guid); + // Emit number of probes in this node + MCOS->emitULEB128IntValue(Probes.size()); + // Emit number of direct inlinees + MCOS->emitULEB128IntValue(Inlinees.size()); + // Emit probes in this group + for (const auto &Probe : Probes) { + Probe.Emit(MCOS, LastProbe); + LastProbe = &Probe; + } + } else { + assert(Probes.empty()); + } + + // Emit descendent + for (const auto &Inlinee : Inlinees) { + if (Guid) { + // Emit probe index + MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first)); + LLVM_DEBUG({ + dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); + dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n"; + }); + } + // Emit the group + Inlinee.second->emit(MCOS, LastProbe); + } + + LLVM_DEBUG({ + MCPseudoProbeTable::DdgPrintIndent -= 2; + dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); + dbgs() << "]\n"; + }); +} + +void MCPseudoProbeSection::Emit(MCObjectStreamer *MCOS) { + MCContext &Ctx = MCOS->getContext(); + + for (auto &ProbeSec : MCProbeDivisions) { + const MCPseudoProbe *LastProbe = nullptr; + if (auto S = + Ctx.getObjectFileInfo()->getPseudoProbeSection(ProbeSec.first)) { + // Switch to the .pseudoprobe section or a comdat group. + MCOS->SwitchSection(S); + // Emit probes grouped by GUID. + ProbeSec.second.emit(MCOS, LastProbe); + } + } +} + +// +// This emits the pseudo probe tables. +// +void MCPseudoProbeTable::Emit(MCObjectStreamer *MCOS) { + MCContext &Ctx = MCOS->getContext(); + auto &ProbeTable = Ctx.getMCPseudoProbeTable(); + + // Bail out early so we don't switch to the pseudo_probe section needlessly + // and in doing so create an unnecessary (if empty) section. + auto &ProbeSections = ProbeTable.getProbeSections(); + if (ProbeSections.empty()) + return; + + LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0); + + // Put out the probe. + ProbeSections.Emit(MCOS); +} diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -22,6 +22,7 @@ #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCPseudoProbe.h" #include "llvm/MC/MCRegister.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSection.h" @@ -1033,6 +1034,25 @@ visitUsedExpr(*Inst.getOperand(i).getExpr()); } +void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, + uint64_t Attr, + const MCPseudoProbeInlineStack &InlineStack) { + auto &Context = getContext(); + + // Create a symbol at in the current section for use in the probe. + MCSymbol *ProbeSym = Context.createTempSymbol(); + + // Set the value of the symbol to use for the MCPseudoProbe. + emitLabel(ProbeSym); + + // Create a (local) probe entry with the symbol. + MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr); + + // Add the probe entry to this section's entries. + Context.getMCPseudoProbeTable().getProbeSections().addPseudoProbe( + getCurrentSectionOnly(), Probe, InlineStack); +} + void MCStreamer::emitAbsoluteSymbolDiff(const MCSymbol *Hi, const MCSymbol *Lo, unsigned Size) { // Get the Hi-Lo expression. diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp --- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp @@ -35,12 +35,46 @@ STATISTIC(ArtificialDbgLine, "Number of probes that have an artificial debug line"); -SampleProfileProber::SampleProfileProber(Function &Func) : F(&Func) { +// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index +// value of each BB in the CFG. The higher 32 bits record the number of edges +// preceded by the number of indirect calls. +// This is derived from FuncPGOInstrumentation::computeCFGHash(). +void SampleProfileProber::computeCFGHash() { + std::vector Indexes; + JamCRC JC; + for (auto &BB : *F) { + auto TI = BB.getTerminator(); + for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) { + auto Succ = TI->getSuccessor(I); + auto Index = getBlockId(Succ); + for (int J = 0; J < 4; J++) + Indexes.push_back((uint8_t)(Index >> (J * 8))); + } + } + + JC.update(Indexes); + + FunctionHash = (uint64_t)CallProbeIds.size() << 48 | + (uint64_t)Indexes.size() << 32 | JC.getCRC(); + // Reserve bit 60-63 for other information purpose. + FunctionHash &= 0x0FFFFFFFFFFFFFFF; + assert(FunctionHash != 0); + LLVM_DEBUG(dbgs() << "\nFunction Hash Computation for " << F->getName() + << ":\n" + << " CRC = " << JC.getCRC() << ", Edges = " + << Indexes.size() << ", ICSites = " << CallProbeIds.size() + << ", Hash = " << FunctionHash << "\n"); +} + +SampleProfileProber::SampleProfileProber(Function &Func, + const std::string &CurModuleUniqueId) + : F(&Func), CurModuleUniqueId(CurModuleUniqueId) { BlockProbeIds.clear(); CallProbeIds.clear(); LastProbeId = (uint32_t)PseudoProbeReservedId::Last; computeProbeIdForBlocks(); computeProbeIdForCallsites(); + computeCFGHash(); } void SampleProfileProber::computeProbeIdForBlocks() { @@ -149,14 +183,50 @@ Call->setDebugLoc(DIL); } } + + // Create module-level metadata that contains function info necessary to + // synthesize probe-based sample counts, which are + // - FunctionGUID + // - FunctionHash. + // - FunctionName + auto Hash = getFunctionHash(); + auto MD = MDB.createPseudoProbeDesc(Guid, Hash, &F); + auto NMD = M->getNamedMetadata(PseudoProbeDescMetadataName); + assert(NMD); + NMD->addOperand(MD); + + // Preserve a comdat group to hold all probes materialized later. This + // allows that when the function is considered dead and removed, the + // materialized probes are disposed too. + // Imported functions are defined in another module. They do not need + // the following handling since same care will be taken for them in their + // original module. The pseudo probes inserted into an imported functions + // above will naturally not be emitted since the imported function is free + // from object emission. However they will be emitted together with the + // inliner functions that the imported function is inlined into. We are not + // creating a comdat group for an import function since it's useless anyway. + if (!F.isDeclarationForLinker()) { + if (TM) { + auto Triple = TM->getTargetTriple(); + if (Triple.supportsCOMDAT() && TM->getFunctionSections()) { + GetOrCreateFunctionComdat(F, Triple, CurModuleUniqueId); + } + } + } } PreservedAnalyses SampleProfileProbePass::run(Module &M, ModuleAnalysisManager &AM) { + auto ModuleId = getUniqueModuleId(&M); + // Create the pseudo probe desc metadata beforehand. + // Note that modules with only data but no functions will require this to + // be set up so that they will be known as probed later. + M.getOrInsertNamedMetadata(PseudoProbeDescMetadataName); + for (auto &F : M) { if (F.isDeclaration()) continue; - SampleProfileProber ProbeManager(F); + SampleProfileProber ProbeManager(F, ModuleId); ProbeManager.instrumentOneFunc(F, TM); } diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll @@ -0,0 +1,95 @@ +; REQUIRES: x86_64-linux +; RUN: opt < %s -passes='pseudo-probe,cgscc(inline)' -function-sections -mtriple=x86_64-unknown-linux-gnu -S -o %t +; RUN: FileCheck %s < %t --check-prefix=CHECK-IL +; RUN: llc -pseudo-probe-for-profiling -function-sections <%t -filetype=asm -o %t1 +; RUN: FileCheck %s < %t1 --check-prefix=CHECK-ASM +; RUN: llc -pseudo-probe-for-profiling -function-sections <%t -filetype=obj -o %t2 +; RUN: llvm-objdump --section-headers %t2 | FileCheck %s --check-prefix=CHECK-OBJ +; RUN: llvm-mc -filetype=asm <%t1 -o %t3 +; RUN: FileCheck %s < %t3 --check-prefix=CHECK-ASM +; RUN: llvm-mc -filetype=obj <%t1 -o %t4 +; RUN: llvm-objdump --section-headers %t4 | FileCheck %s --check-prefix=CHECK-OBJ + +define dso_local void @foo2() !dbg !7 { +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID1:]], i64 1, i32 0), !dbg ![[#]] +; CHECK-ASM: .pseudoprobe [[#GUID1:]] 1 0 0 + ret void, !dbg !10 +} + +define dso_local void @foo() #0 !dbg !11 { +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID2:]], i64 1, i32 0), !dbg ![[#]] +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID1]], i64 1, i32 0), !dbg ![[#DL1:]] +; CHECK-ASM: .pseudoprobe [[#GUID2:]] 1 0 0 +; CHECK-ASM: .pseudoprobe [[#GUID1]] 1 0 0 @ [[#GUID2]]:2 + call void @foo2(), !dbg !12 + ret void, !dbg !13 +} + +define dso_local i32 @entry() !dbg !14 { +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID3:]], i64 1, i32 0), !dbg ![[#]] +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID2]], i64 1, i32 0), !dbg ![[#DL2:]] +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID1]], i64 1, i32 0), !dbg ![[#DL3:]] +; CHECK-ASM: .pseudoprobe [[#GUID3:]] 1 0 0 +; CHECK-ASM: .pseudoprobe [[#GUID2]] 1 0 0 @ [[#GUID3]]:2 +; CHECK-ASM: .pseudoprobe [[#GUID1]] 1 0 0 @ [[#GUID2]]:2 @ [[#GUID3]]:2 + call void @foo(), !dbg !18 + ret i32 0, !dbg !19 +} + + +; CHECK-IL: ![[#SCOPE1:]] = distinct !DISubprogram(name: "foo2" +; CHECK-IL: ![[#SCOPE2:]] = distinct !DISubprogram(name: "foo" +; CHECK-IL: ![[#DL1]] = !DILocation(line: 3, column: 1, scope: ![[#SCOPE1]], inlinedAt: ![[#INL1:]]) +; CHECK-IL: ![[#INL1]] = distinct !DILocation(line: 7, column: 3, scope: ![[#BL1:]]) +;; A discriminator of 134217751 which is 0x8000017 in hexdecimal, stands for a direct call probe +;; with an index of 2. +; CHECK-IL: ![[#BL1]] = !DILexicalBlockFile(scope: ![[#SCOPE2]], file: !1, discriminator: 134217751) +; CHECK-IL: ![[#SCOPE3:]] = distinct !DISubprogram(name: "entry" +; CHECK-IL: ![[#DL2]] = !DILocation(line: 7, column: 3, scope: ![[#SCOPE2]], inlinedAt: ![[#INL2:]]) +; CHECK-IL: ![[#INL2]] = distinct !DILocation(line: 11, column: 3, scope: ![[#BL2:]]) +; CHECK-IL: ![[#BL2]] = !DILexicalBlockFile(scope: ![[#SCOPE3]], file: !1, discriminator: 134217751) +; CHECK-IL: ![[#DL3]] = !DILocation(line: 3, column: 1, scope: ![[#SCOPE1]], inlinedAt: ![[#INL3:]]) +; CHECK-IL: ![[#INL3]] = distinct !DILocation(line: 7, column: 3, scope: ![[#BL1]], inlinedAt: ![[#INL2]]) + + +; Check the generation of .pseudo_probe_desc section +; CHECK-ASM: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo2,comdat +; CHECK-ASM-NEXT: .quad [[#GUID1]] +; CHECK-ASM-NEXT: .quad [[#HASH1:]] +; CHECK-ASM-NEXT: .byte 4 +; CHECK-ASM-NEXT: .ascii "foo2" +; CHECK-ASM-NEXT: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo,comdat +; CHECK-ASM-NEXT: .quad [[#GUID2]] +; CHECK-ASM-NEXT: .quad [[#HASH2:]] +; CHECK-ASM-NEXT: .byte 3 +; CHECK-ASM-NEXT: .ascii "foo" +; CHECK-ASM-NEXT: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_entry,comdat +; CHECK-ASM-NEXT: .quad [[#GUID3]] +; CHECK-ASM-NEXT: .quad [[#HASH3:]] +; CHECK-ASM-NEXT: .byte 5 +; CHECK-ASM-NEXT: .ascii "entry" + +; CHECK-OBJ: .pseudo_probe_desc +; CHECK-OBJ: .pseudo_probe + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug) +!1 = !DIFile(filename: "foo.c", directory: "any") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!7 = distinct !DISubprogram(name: "foo2", scope: !1, file: !1, line: 2, type: !8, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) +!8 = !DISubroutineType(types: !9) +!9 = !{null} +!10 = !DILocation(line: 3, column: 1, scope: !7) +!11 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 6, type: !8, scopeLine: 6, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) +!12 = !DILocation(line: 7, column: 3, scope: !11) +!13 = !DILocation(line: 8, column: 1, scope: !11) +!14 = distinct !DISubprogram(name: "entry", scope: !1, file: !1, line: 10, type: !15, scopeLine: 10, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) +!15 = !DISubroutineType(types: !16) +!16 = !{!17} +!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!18 = !DILocation(line: 11, column: 3, scope: !14) +!19 = !DILocation(line: 12, column: 3, scope: !14) diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll @@ -1,7 +1,14 @@ +; REQUIRES: x86_64-linux ; RUN: opt < %s -passes=pseudo-probe -function-sections -S -o %t ; RUN: FileCheck %s < %t --check-prefix=CHECK-IL ; RUN: llc %t -pseudo-probe-for-profiling -stop-after=pseudo-probe-inserter -o - | FileCheck %s --check-prefix=CHECK-MIR -; +; RUN: llc %t -pseudo-probe-for-profiling -function-sections -filetype=asm -o %t1 +; RUN: FileCheck %s < %t1 --check-prefix=CHECK-ASM +; RUN: llc %t -pseudo-probe-for-profiling -function-sections -filetype=obj -o %t2 +; RUN: llvm-objdump --section-headers %t2 | FileCheck %s --check-prefix=CHECK-OBJ +; RUN: llvm-mc %t1 -filetype=obj -o %t3 +; RUN: llvm-objdump --section-headers %t3 | FileCheck %s --check-prefix=CHECK-OBJ + ;; Check the generation of pseudoprobe intrinsic call. define void @foo(i32 %x) !dbg !3 { @@ -9,18 +16,23 @@ %cmp = icmp eq i32 %x, 0 ; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 1, i32 0), !dbg ![[#FAKELINE:]] ; CHECK-MIR: PSEUDO_PROBE [[#GUID:]], 1, 0, 0 +; CHECK-ASM: .pseudoprobe [[#GUID:]] 1 0 0 br i1 %cmp, label %bb1, label %bb2 bb1: ; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 2, i32 0), !dbg ![[#FAKELINE]] ; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 3, 0, 0 ; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0, 0 +; CHECK-ASM: .pseudoprobe [[#GUID]] 3 0 0 +; CHECK-ASM: .pseudoprobe [[#GUID]] 4 0 0 br label %bb3 bb2: ; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 3, i32 0), !dbg ![[#FAKELINE]] ; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 2, 0, 0 ; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0, 0 +; CHECK-ASM: .pseudoprobe [[#GUID]] 2 0 0 +; CHECK-ASM: .pseudoprobe [[#GUID]] 4 0 0 br label %bb3 bb3: @@ -34,13 +46,16 @@ entry: ; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID2:]], i64 1, i32 0) ; CHECK-MIR: PSEUDO_PROBE [[#GUID2:]], 1, 0, 0 +; CHECK-ASM: .pseudoprobe [[#GUID2:]] 1 0 0 ; Check pseudo_probe metadata attached to the indirect call instruction. ; CHECK-IL: call void %f(i32 1), !dbg ![[#PROBE0:]] ; CHECK-MIR: PSEUDO_PROBE [[#GUID2]], 2, 1, 0 +; CHECK-ASM: .pseudoprobe [[#GUID2]] 2 1 0 call void %f(i32 1), !dbg !13 ; Check pseudo_probe metadata attached to the direct call instruction. ; CHECK-IL: call void @bar(i32 1), !dbg ![[#PROBE1:]] ; CHECK-MIR: PSEUDO_PROBE [[#GUID2]], 3, 2, 0 +; CHECK-ASM: .pseudoprobe [[#GUID2]] 3 2 0 call void @bar(i32 1) ret void } @@ -57,6 +72,20 @@ ;; with an index of 3. ; CHECK-IL: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 134217759) +; Check the generation of .pseudo_probe_desc section +; CHECK-ASM: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo,comdat +; CHECK-ASM-NEXT: .quad [[#GUID]] +; CHECK-ASM-NEXT: .quad [[#HASH:]] +; CHECK-ASM-NEXT: .byte 3 +; CHECK-ASM-NEXT: .ascii "foo" +; CHECK-ASM-NEXT: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo2,comdat +; CHECK-ASM-NEXT: .quad [[#GUID2]] +; CHECK-ASM-NEXT: .quad [[#HASH2:]] +; CHECK-ASM-NEXT: .byte 4 +; CHECK-ASM-NEXT: .ascii "foo2" + +; CHECK-OBJ-COUNT-2: .pseudo_probe_desc +; CHECK-OBJ-COUNT-2: .pseudo_probe !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!9, !10}