diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1086,6 +1086,10 @@ /// known nonzero constant. The only operand here is the chain. GET_DYNAMIC_AREA_OFFSET, + /// Pseudo probe for AutoFDO, as a place holder in a basic block to improve + /// the sample counts quality. + PSEUDO_PROBE, + /// VSCALE(IMM) - Returns the runtime scaling factor used to calculate the /// number of elements within a scalable vector. IMM is a constant integer /// multiplier that is applied to the runtime value. diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -1241,6 +1241,7 @@ case TargetOpcode::DBG_LABEL: case TargetOpcode::LIFETIME_START: case TargetOpcode::LIFETIME_END: + case TargetOpcode::PSEUDO_PROBE: return true; } } diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1169,6 +1169,11 @@ SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset = -1); + /// Creates a PseudoProbeSDNode with function GUID `Guid` and + /// the index of the block `Index` it is probing. + SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, + uint64_t Index); + /// Create a MERGE_VALUES node from the given operands. SDValue getMergeValues(ArrayRef Ops, const SDLoc &dl); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1794,6 +1794,29 @@ } }; +/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and +/// the index of the basic block being probed. A pseudo probe serves as a place +/// holder and will be removed at the end of compilation. It does not have any +/// operand because we do not want the instruction selection to deal with any. +class PseudoProbeSDNode : public SDNode { + friend class SelectionDAG; + uint64_t Guid; + uint64_t Index; + + PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl, + SDVTList VTs, uint64_t Guid, uint64_t Index) + : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index) {} + +public: + int64_t getGuid() const { return Guid; } + uint64_t getIndex() const { return Index; } + + // Methods to support isa and dyn_cast + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::PSEUDO_PROBE; + } +}; + class JumpTableSDNode : public SDNode { friend class SelectionDAG; diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -106,6 +106,9 @@ HANDLE_TARGET_OPCODE(LIFETIME_START) HANDLE_TARGET_OPCODE(LIFETIME_END) +/// Pseudo probe +HANDLE_TARGET_OPCODE(PSEUDO_PROBE) + /// A Stackmap instruction captures the location of live variables at its /// position in the instruction stream. It is followed by a shadow of bytes /// that must lie within the function and not contain another stackmap. diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -1139,6 +1139,13 @@ let AsmString = "LIFETIME_END"; let hasSideEffects = 0; } +def PSEUDO_PROBE : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i64imm:$guid, i64imm:$index, i64imm:$type); + let AsmString = "PSEUDO_PROBE"; + let hasSideEffects = 1; +} + def STACKMAP : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i64imm:$id, i32imm:$nbytes, variable_ops); diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -1082,6 +1082,18 @@ break; } + case ISD::PSEUDO_PROBE: { + unsigned TarOp = TargetOpcode::PSEUDO_PROBE; + auto Guid = cast(Node)->getGuid(); + auto Index = cast(Node)->getIndex(); + + BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp)) + .addImm(Guid) + .addImm(Index) + .addImm(0); // 0 for block probes + break; + } + case ISD::INLINEASM: case ISD::INLINEASM_BR: { unsigned NumOps = Node->getNumOperands(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -537,6 +537,10 @@ ID.AddInteger(cast(N)->getOffset()); } break; + case ISD::PSEUDO_PROBE: + ID.AddInteger(cast(N)->getGuid()); + ID.AddInteger(cast(N)->getIndex()); + break; case ISD::JumpTable: case ISD::TargetJumpTable: ID.AddInteger(cast(N)->getIndex()); @@ -6898,6 +6902,30 @@ return V; } +SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, + uint64_t Guid, uint64_t Index) { + + const unsigned Opcode = ISD::PSEUDO_PROBE; + const auto VTs = getVTList(MVT::Other); + SDValue Ops[] = {Chain}; + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTs, Ops); + ID.AddInteger(Guid); + ID.AddInteger(Index); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) + return SDValue(E, 0); + + auto *N = newSDNode(Opcode, Dl.getIROrder(), + Dl.getDebugLoc(), VTs, Guid, Index); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); + InsertNode(N); + SDValue V(N, 0); + NewSDValueDbgMsg(V, "Creating new node: ", this); + return V; +} + /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a /// MachinePointerInfo record from it. This is particularly useful because the /// code generator has many cases where it doesn't bother passing in a diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6615,6 +6615,13 @@ } return; } + case Intrinsic::pseudoprobe: { + auto Guid = cast(I.getArgOperand(0))->getZExtValue(); + auto Index = cast(I.getArgOperand(1))->getZExtValue(); + Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index); + DAG.setRoot(Res); + return; + } case Intrinsic::invariant_start: // Discard region information. setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout()))); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -394,6 +394,8 @@ case ISD::DEBUGTRAP: return "debugtrap"; case ISD::LIFETIME_START: return "lifetime.start"; case ISD::LIFETIME_END: return "lifetime.end"; + case ISD::PSEUDO_PROBE: + return "pseudoprobe"; case ISD::GC_TRANSITION_START: return "gc_transition.start"; case ISD::GC_TRANSITION_END: return "gc_transition.end"; case ISD::GET_DYNAMIC_AREA_OFFSET: return "get.dynamic.area.offset"; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2792,6 +2792,7 @@ case ISD::ANNOTATION_LABEL: case ISD::LIFETIME_START: case ISD::LIFETIME_END: + case ISD::PSEUDO_PROBE: NodeToMatch->setNodeId(-1); // Mark selected. return; case ISD::AssertSext: diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe.mir b/llvm/test/Transforms/SampleProfile/pseudo-probe.mir new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe.mir @@ -0,0 +1,29 @@ + +# REQUIRES: x86-registered-target +# Ensure llc can read and parse MIR pseudo probe operations. +# RUN: llc -O0 -mtriple x86_64-- -run-pass none %s -o - | FileCheck %s + +# CHECK: PSEUDO_PROBE 6699318081062747564, 1, 0 +# CHECK: PSEUDO_PROBE 6699318081062747564, 3, 0 +# CHECK: PSEUDO_PROBE 6699318081062747564, 4, 0 +# CHECK: PSEUDO_PROBE 6699318081062747564, 2, 0 +# CHECK: PSEUDO_PROBE 6699318081062747564, 4, 0 + +name: foo +body: | + bb.0: + TEST32rr killed renamable $edi, renamable $edi, implicit-def $eflags + PSEUDO_PROBE 6699318081062747564, 1, 0 + JCC_1 %bb.1, 4, implicit $eflags + + bb.2: + PSEUDO_PROBE 6699318081062747564, 3, 0 + PSEUDO_PROBE 6699318081062747564, 4, 0 + RETQ + + bb.1: + PSEUDO_PROBE 6699318081062747564, 2, 0 + PSEUDO_PROBE 6699318081062747564, 4, 0 + RETQ + +...