Index: llvm/include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- llvm/include/llvm/CodeGen/ISDOpcodes.h +++ llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1285,6 +1285,11 @@ VECREDUCE_UMAX, VECREDUCE_UMIN, + // The `llvm.experimental.stackmap` intrinsic. + // Operands: input chain, glue, , , [live0[, live1...]] + // Outputs: output chain, glue + STACKMAP, + // Vector Predication #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID, #include "llvm/IR/VPIntrinsics.def" Index: llvm/include/llvm/CodeGen/SelectionDAGISel.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -321,6 +321,7 @@ void Select_FREEZE(SDNode *N); void Select_ARITH_FENCE(SDNode *N); + void Select_STACKMAP(SDNode *N); private: void DoInstructionSelection(); Index: llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2915,6 +2915,9 @@ case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; + case ISD::STACKMAP: + Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); + break; } if (!Res.getNode()) @@ -3042,3 +3045,17 @@ return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), ST->getMemOperand()); } + +SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { + assert(OpNo > 1); // Because the first two arguments are guaranteed legal. + SmallVector NewOps(N->ops().begin(), N->ops().end()); + SDValue Op = N->getOperand(OpNo); + NewOps[OpNo] = GetSoftPromotedHalf(Op); + SDValue NewNode = + DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); + + for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) + ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); + + return SDValue(); // Signal that we replaced the node ourselves. +} Index: llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -19,6 +19,7 @@ #include "LegalizeTypes.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" @@ -1723,6 +1724,9 @@ break; case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; + case ISD::STACKMAP: + Res = PromoteIntOp_STACKMAP(N, OpNo); + break; } // If the result is null, the sub-method took care of registering results etc. @@ -2304,6 +2308,16 @@ return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); } +SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { + assert(OpNo > 1); // Because the first two arguments are guaranteed legal. + SmallVector NewOps(N->ops().begin(), N->ops().end()); + SDValue Operand = N->getOperand(OpNo); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); + NewOps[OpNo] = + DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, N->getOperand(OpNo)); + return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); +} + //===----------------------------------------------------------------------===// // Integer Result Expansion //===----------------------------------------------------------------------===// @@ -4653,6 +4667,9 @@ case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; + case ISD::STACKMAP: + Res = ExpandIntOp_STACKMAP(N, OpNo); + break; } // If the result is null, the sub-method took care of registering results etc. @@ -5481,3 +5498,44 @@ return DAG.getBuildVector(N->getValueType(0), dl, NewOps); } + +SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { + assert(OpNo > 1); + + SDValue Op = N->getOperand(OpNo); + SDLoc DL = SDLoc(N); + SmallVector NewOps; + + // Copy operands before the one being expanded. + for (unsigned I = 0; I < OpNo; I++) + NewOps.push_back(N->getOperand(I)); + + if (Op->getOpcode() == ISD::Constant) { + ConstantSDNode *CN = cast(Op); + EVT Ty = Op.getValueType(); + if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { + NewOps.push_back( + DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); + NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); + } else { + // FIXME: https://github.com/llvm/llvm-project/issues/55609 + return SDValue(); + } + } else { + // FIXME: Non-constant operands are not yet handled: + // - https://github.com/llvm/llvm-project/issues/26431 + // - https://github.com/llvm/llvm-project/issues/55957 + return SDValue(); + } + + // Copy remaining operands. + for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) + NewOps.push_back(N->getOperand(I)); + + SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); + + for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) + ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); + + return SDValue(); // Signal that we have replaced the node already. +} Index: llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -402,6 +402,7 @@ SDValue PromoteIntOp_VECREDUCE(SDNode *N); SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_SET_ROUNDING(SDNode *N); + SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo); void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code); @@ -493,6 +494,7 @@ SDValue ExpandIntOp_RETURNADDR(SDNode *N); SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N); SDValue ExpandIntOp_SPLAT_VECTOR(SDNode *N); + SDValue ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo); void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &dl); @@ -741,6 +743,7 @@ SDValue SoftPromoteHalfOp_SETCC(SDNode *N); SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo); SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo); + SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo); //===--------------------------------------------------------------------===// // Scalarization Support: LegalizeVectorTypes.cpp Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -9392,9 +9392,9 @@ } } -/// Lower llvm.experimental.stackmap directly to its target opcode. +/// Lower llvm.experimental.stackmap. void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { - // void @llvm.experimental.stackmap(i32 , i32 , + // void @llvm.experimental.stackmap(i64 , i32 , // [live variables...]) assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value."); @@ -9419,29 +9419,45 @@ Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL); InFlag = Chain.getValue(1); - // Add the and constants. - SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); - Ops.push_back(DAG.getTargetConstant( - cast(IDVal)->getZExtValue(), DL, MVT::i64)); - SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); - Ops.push_back(DAG.getTargetConstant( - cast(NBytesVal)->getZExtValue(), DL, - MVT::i32)); - - // Push live variables for the stack map. - addStackMapLiveVars(CI, 2, DL, Ops, *this); - - // We are not pushing any register mask info here on the operands list, - // because the stackmap doesn't clobber anything. - - // Push the chain and the glue flag. + // Add the STACKMAP operands, starting with DAG house-keeping. Ops.push_back(Chain); Ops.push_back(InFlag); + // Add the , operands. + // + // These do not require legalisation, and can be emitted directly to target + // constant nodes. + SDValue ID = getValue(CI.getArgOperand(0)); + assert(ID.getValueType() == MVT::i64); + SDValue IDConst = DAG.getTargetConstant( + cast(ID)->getZExtValue(), DL, ID.getValueType()); + Ops.push_back(IDConst); + + SDValue Shad = getValue(CI.getArgOperand(1)); + assert(Shad.getValueType() == MVT::i32); + SDValue ShadConst = DAG.getTargetConstant( + cast(Shad)->getZExtValue(), DL, Shad.getValueType()); + Ops.push_back(ShadConst); + + // Add the live variables. + for (unsigned I = 2; I < CI.arg_size(); I++) { + SDValue Op = getValue(CI.getArgOperand(I)); + + // Things on the stack are pointer-typed, meaning that they are already + // legal and can be emitted directly to target nodes. + if (FrameIndexSDNode *FI = dyn_cast(Op)) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + Ops.push_back(DAG.getTargetFrameIndex( + FI->getIndex(), TLI.getFrameIndexTy(DAG.getDataLayout()))); + } else { + // Otherwise emit a target independent node to be legalised. + Ops.push_back(getValue(CI.getArgOperand(I))); + } + } + // Create the STACKMAP node. SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); - SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops); - Chain = SDValue(SM, 0); + Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops); InFlag = Chain.getValue(1); Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL); Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -486,6 +486,8 @@ case ISD::VECREDUCE_UMIN: return "vecreduce_umin"; case ISD::VECREDUCE_FMAX: return "vecreduce_fmax"; case ISD::VECREDUCE_FMIN: return "vecreduce_fmin"; + case ISD::STACKMAP: + return "stackmap"; // Vector Predication #define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...) \ Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -51,6 +51,7 @@ #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/StackProtector.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -2241,6 +2242,52 @@ N->getOperand(0)); } +void SelectionDAGISel::Select_STACKMAP(SDNode *N) { + std::vector Ops; + auto *It = N->op_begin(); + SDLoc DL(N); + + // Stash the chain and glue operands so we can move them to the end. + SDValue Chain = *It++; + SDValue InFlag = *It++; + + // operand. + SDValue ID = *It++; + assert(ID.getValueType() == MVT::i64); + Ops.push_back(ID); + + // operand. + SDValue Shad = *It++; + assert(Shad.getValueType() == MVT::i32); + Ops.push_back(Shad); + + // Live variable operands. + for (; It != N->op_end(); It++) { + SDNode *OpNode = It->getNode(); + SDValue O; + + // FrameIndex nodes should have been directly emitted to TargetFrameIndex + // nodes at DAG-construction time. + assert(OpNode->getOpcode() != ISD::FrameIndex); + + if (OpNode->getOpcode() == ISD::Constant) { + Ops.push_back( + CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); + O = CurDAG->getTargetConstant( + cast(OpNode)->getZExtValue(), DL, It->getValueType()); + } else { + O = *It; + } + Ops.push_back(O); + } + + Ops.push_back(Chain); + Ops.push_back(InFlag); + + SDVTList NodeTys = CurDAG->getVTList(MVT::Other, MVT::Glue); + CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops); +} + /// GetVBR - decode a vbr encoding whose top bit is set. LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { @@ -2795,6 +2842,9 @@ case ISD::ARITH_FENCE: Select_ARITH_FENCE(NodeToMatch); return; + case ISD::STACKMAP: + Select_STACKMAP(NodeToMatch); + return; } assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); Index: llvm/test/CodeGen/X86/selectiondag-stackmap-legalize.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/selectiondag-stackmap-legalize.ll @@ -0,0 +1,152 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -enable-patchpoint-liveness=false | FileCheck %s + +; CHECK-LABEL: .section __LLVM_STACKMAPS,__llvm_stackmaps +; CHECK-NEXT: __LLVM_StackMaps: + +; Header +; CHECK-NEXT: .byte 3 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 0 + +; NumFunctions +; CHECK-NEXT: .long 1 +; NumConstants +; CHECK-NEXT: .long 0 +; NumRecords +; CHECK-NEXT: .long 1 + +; StackSizeRecord[NumFunctions] +; StackSizeRecord[0] +; CHECK-NEXT: .quad _main +; CHECK-NEXT: .quad 24 +; CHECK-NEXT: .quad 1 + +; Constants[NumConstants] (empty) + +; StkMapRecord[NumRecords] +; StkMapRecord[0] +; CHECK-NEXT: .quad 0 +; CHECK-NEXT: .long {{.*}} +; CHECK-NEXT: .short {{.*}} +; NumLocations +; CHECK-NEXT: .short 11 +; Location[NumLocations] +; Location[0] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 1 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[1] +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 22 +; Location[2] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 1 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[3] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 16 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[4] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 16 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[5] +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 8 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 66 +; Location[6] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 4 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[7] +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[8] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 4 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[9] +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[10] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 1 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 + +@p32 = external global i8 addrspace(270)* + +%struct1 = type {i32, i64} +%struct2 = type {i1, i1, i1} + +declare void @llvm.experimental.stackmap(i64, i32, ...) + +define dso_local i32 @main(i32 %argc, i8** %argv) { +entry: + %i1reg = icmp eq i32 %argc, 5 + %i7reg = zext i1 %i1reg to i7 + %halfreg = sitofp i32 %argc to half + %ptr32 = load i8 addrspace(270)*, i8 addrspace(270)** @p32 + %structreg1 = insertvalue %struct1 zeroinitializer, i32 %argc, 0 + %structreg2 = insertvalue %struct2 zeroinitializer, i1 %i1reg, 0 + call void (i64, i32, ...) @llvm.experimental.stackmap( + i64 0, + i32 0, + i1 %i1reg, + i7 22, + i7 %i7reg, + half 1.0, + half %halfreg, + i128 66, + ; FIXME: fix and test vectors. At the moment even legally sized vectors + ; are broken: + ; https://github.com/llvm/llvm-project/issues/55613 + ; + ; FIXME: test non-constant i128 once these are fixed: + ; - https://github.com/llvm/llvm-project/issues/26431 + ; - https://github.com/llvm/llvm-project/issues/55957 + i8 addrspace(270)* %ptr32, + ; FIXME: The stackmap record generated for structs is incorrect: + ; - https://github.com/llvm/llvm-project/issues/55649 + ; - https://github.com/llvm/llvm-project/issues/55957 + %struct1 zeroinitializer, + %struct1 %structreg1, + %struct2 zeroinitializer, + %struct2 %structreg2) + ret i32 0 +}