Index: llvm/include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- llvm/include/llvm/CodeGen/ISDOpcodes.h +++ llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1278,6 +1278,11 @@ VECREDUCE_UMAX, VECREDUCE_UMIN, + // The `llvm.experimental.stackmap` intrinsic. + // Operands: input chain, glue, , , [live0[, live1...]] + // Outputs: output chain, glue + STACKMAP, + // Vector Predication #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID, #include "llvm/IR/VPIntrinsics.def" Index: llvm/include/llvm/CodeGen/SelectionDAGISel.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -321,6 +321,7 @@ void Select_FREEZE(SDNode *N); void Select_ARITH_FENCE(SDNode *N); + void Select_STACKMAP(SDNode *N); private: void DoInstructionSelection(); Index: llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2907,6 +2907,9 @@ case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; + case ISD::STACKMAP: + Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); + break; } if (!Res.getNode()) @@ -2914,10 +2917,12 @@ assert(Res.getNode() != N && "Expected a new node!"); - assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && + assert(Res.getValueType() == N->getValueType(0) && "Invalid operand expansion"); - ReplaceValueWith(SDValue(N, 0), Res); + for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) + ReplaceValueWith(SDValue(N, ResNum), Res.getValue(ResNum)); + return false; } @@ -3034,3 +3039,11 @@ return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), ST->getMemOperand()); } + +SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { + assert(OpNo > 1); // Because the first two arguments are guaranteed legal. + SmallVector NewOps(N->ops().begin(), N->ops().end()); + SDValue Op = N->getOperand(OpNo); + NewOps[OpNo] = GetSoftPromotedHalf(Op); + return DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); +} Index: llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1689,6 +1689,9 @@ break; case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; + case ISD::STACKMAP: + Res = PromoteIntOp_STACKMAP(N, OpNo); + break; } // If the result is null, the sub-method took care of registering results etc. @@ -2265,6 +2268,13 @@ return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); } +SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { + assert(OpNo > 1); // Because the first two arguments are guaranteed legal. + SmallVector NewOps(N->ops().begin(), N->ops().end()); + NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); + return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); +} + //===----------------------------------------------------------------------===// // Integer Result Expansion //===----------------------------------------------------------------------===// Index: llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -404,6 +404,7 @@ SDValue PromoteIntOp_VECREDUCE(SDNode *N); SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_SET_ROUNDING(SDNode *N); + SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo); void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code); @@ -743,6 +744,7 @@ SDValue SoftPromoteHalfOp_SETCC(SDNode *N); SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo); SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo); + SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo); //===--------------------------------------------------------------------===// // Scalarization Support: LegalizeVectorTypes.cpp Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -9376,9 +9376,9 @@ } } -/// Lower llvm.experimental.stackmap directly to its target opcode. +/// Lower llvm.experimental.stackmap. void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { - // void @llvm.experimental.stackmap(i32 , i32 , + // void @llvm.experimental.stackmap(i64 , i32 , // [live variables...]) assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value."); @@ -9403,29 +9403,46 @@ Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL); InFlag = Chain.getValue(1); - // Add the and constants. - SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); - Ops.push_back(DAG.getTargetConstant( - cast(IDVal)->getZExtValue(), DL, MVT::i64)); - SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); - Ops.push_back(DAG.getTargetConstant( - cast(NBytesVal)->getZExtValue(), DL, - MVT::i32)); - - // Push live variables for the stack map. - addStackMapLiveVars(CI, 2, DL, Ops, *this); + // Add the STACKMAP operands, starting with DAG house-keeping. + std::vector Operands; + Operands.push_back(Chain); + Operands.push_back(InFlag); - // We are not pushing any register mask info here on the operands list, - // because the stackmap doesn't clobber anything. - - // Push the chain and the glue flag. - Ops.push_back(Chain); - Ops.push_back(InFlag); + // Add the , operands. + // + // These do not require legalisation, and can be emitted directly to target + // constant nodes. + SDValue ID = getValue(CI.getArgOperand(0)); + assert(ID.getValueType() == MVT::i64); + SDValue IDConst = DAG.getTargetConstant( + cast(ID)->getZExtValue(), DL, ID.getValueType()); + Operands.push_back(IDConst); + + SDValue Shad = getValue(CI.getArgOperand(1)); + assert(Shad.getValueType() == MVT::i32); + SDValue ShadConst = DAG.getTargetConstant( + cast(Shad)->getZExtValue(), DL, Shad.getValueType()); + Operands.push_back(ShadConst); + + // Add the live variables. + for (unsigned I = 2; I < CI.arg_size(); I++) { + SDValue Op = getValue(CI.getArgOperand(I)); + + // Things on the stack are pointer-typed, meaning that they are already + // legal and can be emitted directly to target nodes. + if (FrameIndexSDNode *FI = dyn_cast(Op)) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + Operands.push_back(DAG.getTargetFrameIndex( + FI->getIndex(), TLI.getFrameIndexTy(DAG.getDataLayout()))); + } else { + // Otherwise emit a target independent node to be legalised. + Operands.push_back(getValue(CI.getArgOperand(I))); + } + } // Create the STACKMAP node. SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); - SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops); - Chain = SDValue(SM, 0); + Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Operands); InFlag = Chain.getValue(1); Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL); Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -484,6 +484,8 @@ case ISD::VECREDUCE_UMIN: return "vecreduce_umin"; case ISD::VECREDUCE_FMAX: return "vecreduce_fmax"; case ISD::VECREDUCE_FMIN: return "vecreduce_fmin"; + case ISD::STACKMAP: + return "stackmap"; // Vector Predication #define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...) \ Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -51,6 +51,7 @@ #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/StackProtector.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -2241,6 +2242,57 @@ N->getOperand(0)); } +void SelectionDAGISel::Select_STACKMAP(SDNode *N) { + std::vector Ops; + auto *It = N->op_begin(); + SDLoc DL(N); + + // Stash the chain and glue operands so we can move them to the end. + SDValue Chain = *It++; + SDValue InFlag = *It++; + + // operand. + SDValue ID = *It; + assert(ID.getValueType() == MVT::i64); + Ops.push_back(ID); + It++; + + // operand. + SDValue Shad = *It; + assert(Shad.getValueType() == MVT::i32); + Ops.push_back(Shad); + It++; + + // Live variable operands. + for (; It != N->op_end(); It++) { + SDNode *OpNode = It->getNode(); + SDValue O; + + // FrameIndex nodes should have been directly emitted to TargetFrameIndex + // nodes at DAG-construction time. + assert(OpNode->getOpcode() != ISD::FrameIndex); + + if (OpNode->getOpcode() == ISD::Constant) { + Ops.push_back( + CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); + O = CurDAG->getTargetConstant( + cast(OpNode)->getZExtValue(), DL, It->getValueType()); + } else if (OpNode->getOpcode() == ISD::FrameIndex) { + FrameIndexSDNode *FI = cast(OpNode); + O = CurDAG->getTargetFrameIndex(FI->getIndex(), It->getValueType()); + } else { + O = *It; + } + Ops.push_back(O); + } + + Ops.push_back(Chain); + Ops.push_back(InFlag); + + SDVTList NodeTys = CurDAG->getVTList(MVT::Other, MVT::Glue); + CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops); +} + /// GetVBR - decode a vbr encoding whose top bit is set. LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { @@ -2795,6 +2847,9 @@ case ISD::ARITH_FENCE: Select_ARITH_FENCE(NodeToMatch); return; + case ISD::STACKMAP: + Select_STACKMAP(NodeToMatch); + return; } assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); Index: llvm/test/CodeGen/X86/selectiondag-stackmap-legalize.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/selectiondag-stackmap-legalize.ll @@ -0,0 +1,57 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -enable-patchpoint-liveness=false | FileCheck %s + +; CHECK-LABEL: .section __LLVM_STACKMAPS,__llvm_stackmaps +; CHECK-NEXT: __LLVM_StackMaps: + +; Header +; CHECK-NEXT: .byte 3 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 0 + +; NumFunctions +; CHECK-NEXT: .long 1 +; NumConstants +; CHECK-NEXT: .long 0 +; NumRecords +; CHECK-NEXT: .long 1 + +; StackSizeRecord[NumFunctions] +; StackSizeRecord[0] +; CHECK-NEXT: .quad _main +; CHECK-NEXT: .quad 8 +; CHECK-NEXT: .quad 1 + +; Constants[NumConstants] (empty) + +; StkMapRecord[NumRecords] +; StkMapRecord[0] +; CHECK-NEXT: .quad 0 +; CHECK-NEXT: .long {{.*}} +; CHECK-NEXT: .short {{.*}} +; NumLocations +; CHECK-NEXT: .short 2 +; Location[NumLocations] +; Location[0] +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 1 +; CHECK-NEXT: .short {{.*}} +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; Location[1] +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 22 + + +declare void @llvm.experimental.stackmap(i64, i32, ...) + +define dso_local i32 @main(i32 %argc, i8** %argv) { +entry: + %x = icmp eq i32 %argc, 5 + call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0, i1 %x, i7 22) + ret i32 0 +}