Index: llvm/lib/Target/AMDGPU/SIISelLowering.h =================================================================== --- llvm/lib/Target/AMDGPU/SIISelLowering.h +++ llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -63,6 +63,11 @@ SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset, SDValue CachePolicy, SelectionDAG &DAG) const; + SDValue lowerRawBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG, + unsigned NewOpcode) const; + SDValue lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG, + unsigned NewOpcode) const; + SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; Index: llvm/lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -6663,6 +6663,61 @@ } } +SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op, + SelectionDAG &DAG, + unsigned NewOpcode) const { + SDLoc DL(Op); + + SDValue VData = Op.getOperand(2); + auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); + SDValue Ops[] = { + Op.getOperand(0), // Chain + VData, // vdata + Op.getOperand(3), // rsrc + DAG.getConstant(0, DL, MVT::i32), // vindex + Offsets.first, // voffset + Op.getOperand(5), // soffset + Offsets.second, // offset + Op.getOperand(6), // cachepolicy + DAG.getTargetConstant(0, DL, MVT::i1), // idxen + }; + + auto *M = cast(Op); + M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6])); + + EVT MemVT = VData.getValueType(); + return DAG.getMemIntrinsicNode(NewOpcode, DL, Op->getVTList(), Ops, MemVT, + M->getMemOperand()); +} + +SDValue +SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG, + unsigned NewOpcode) const { + SDLoc DL(Op); + + SDValue VData = Op.getOperand(2); + auto Offsets = splitBufferOffsets(Op.getOperand(5), DAG); + SDValue Ops[] = { + Op.getOperand(0), // Chain + VData, // vdata + Op.getOperand(3), // rsrc + Op.getOperand(4), // vindex + Offsets.first, // voffset + Op.getOperand(6), // soffset + Offsets.second, // offset + Op.getOperand(7), // cachepolicy + DAG.getTargetConstant(1, DL, MVT::i1), // idxen + }; + + auto *M = cast(Op); + M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6], + Ops[3])); + + EVT MemVT = VData.getValueType(); + return DAG.getMemIntrinsicNode(NewOpcode, DL, Op->getVTList(), Ops, MemVT, + M->getMemOperand()); +} + SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntrID = cast(Op.getOperand(1))->getZExtValue(); @@ -7005,154 +7060,60 @@ M->getMemOperand()); } case Intrinsic::amdgcn_raw_buffer_atomic_swap: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SWAP); case Intrinsic::amdgcn_raw_buffer_atomic_add: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_ADD); case Intrinsic::amdgcn_raw_buffer_atomic_sub: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SUB); case Intrinsic::amdgcn_raw_buffer_atomic_smin: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMIN); case Intrinsic::amdgcn_raw_buffer_atomic_umin: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMIN); case Intrinsic::amdgcn_raw_buffer_atomic_smax: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMAX); case Intrinsic::amdgcn_raw_buffer_atomic_umax: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMAX); case Intrinsic::amdgcn_raw_buffer_atomic_and: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_AND); case Intrinsic::amdgcn_raw_buffer_atomic_or: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_OR); case Intrinsic::amdgcn_raw_buffer_atomic_xor: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_XOR); case Intrinsic::amdgcn_raw_buffer_atomic_inc: - case Intrinsic::amdgcn_raw_buffer_atomic_dec: { - auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); - SDValue Ops[] = { - Op.getOperand(0), // Chain - Op.getOperand(2), // vdata - Op.getOperand(3), // rsrc - DAG.getConstant(0, DL, MVT::i32), // vindex - Offsets.first, // voffset - Op.getOperand(5), // soffset - Offsets.second, // offset - Op.getOperand(6), // cachepolicy - DAG.getTargetConstant(0, DL, MVT::i1), // idxen - }; - EVT VT = Op.getValueType(); + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_INC); + case Intrinsic::amdgcn_raw_buffer_atomic_dec: + return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_DEC); - auto *M = cast(Op); - M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6])); - unsigned Opcode = 0; - - switch (IntrID) { - case Intrinsic::amdgcn_raw_buffer_atomic_swap: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SWAP; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_add: - Opcode = AMDGPUISD::BUFFER_ATOMIC_ADD; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_sub: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SUB; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_smin: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SMIN; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_umin: - Opcode = AMDGPUISD::BUFFER_ATOMIC_UMIN; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_smax: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SMAX; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_umax: - Opcode = AMDGPUISD::BUFFER_ATOMIC_UMAX; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_and: - Opcode = AMDGPUISD::BUFFER_ATOMIC_AND; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_or: - Opcode = AMDGPUISD::BUFFER_ATOMIC_OR; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_xor: - Opcode = AMDGPUISD::BUFFER_ATOMIC_XOR; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_inc: - Opcode = AMDGPUISD::BUFFER_ATOMIC_INC; - break; - case Intrinsic::amdgcn_raw_buffer_atomic_dec: - Opcode = AMDGPUISD::BUFFER_ATOMIC_DEC; - break; - default: - llvm_unreachable("unhandled atomic opcode"); - } - - return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT, - M->getMemOperand()); - } case Intrinsic::amdgcn_struct_buffer_atomic_swap: + return lowerStructBufferAtomicIntrin(Op, DAG, + AMDGPUISD::BUFFER_ATOMIC_SWAP); case Intrinsic::amdgcn_struct_buffer_atomic_add: + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_ADD); case Intrinsic::amdgcn_struct_buffer_atomic_sub: + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SUB); case Intrinsic::amdgcn_struct_buffer_atomic_smin: + return lowerStructBufferAtomicIntrin(Op, DAG, + AMDGPUISD::BUFFER_ATOMIC_SMIN); case Intrinsic::amdgcn_struct_buffer_atomic_umin: + return lowerStructBufferAtomicIntrin(Op, DAG, + AMDGPUISD::BUFFER_ATOMIC_UMIN); case Intrinsic::amdgcn_struct_buffer_atomic_smax: + return lowerStructBufferAtomicIntrin(Op, DAG, + AMDGPUISD::BUFFER_ATOMIC_SMAX); case Intrinsic::amdgcn_struct_buffer_atomic_umax: + return lowerStructBufferAtomicIntrin(Op, DAG, + AMDGPUISD::BUFFER_ATOMIC_UMAX); case Intrinsic::amdgcn_struct_buffer_atomic_and: + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_AND); case Intrinsic::amdgcn_struct_buffer_atomic_or: + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_OR); case Intrinsic::amdgcn_struct_buffer_atomic_xor: + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_XOR); case Intrinsic::amdgcn_struct_buffer_atomic_inc: - case Intrinsic::amdgcn_struct_buffer_atomic_dec: { - auto Offsets = splitBufferOffsets(Op.getOperand(5), DAG); - SDValue Ops[] = { - Op.getOperand(0), // Chain - Op.getOperand(2), // vdata - Op.getOperand(3), // rsrc - Op.getOperand(4), // vindex - Offsets.first, // voffset - Op.getOperand(6), // soffset - Offsets.second, // offset - Op.getOperand(7), // cachepolicy - DAG.getTargetConstant(1, DL, MVT::i1), // idxen - }; - EVT VT = Op.getValueType(); - - auto *M = cast(Op); - M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6], - Ops[3])); - unsigned Opcode = 0; + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_INC); + case Intrinsic::amdgcn_struct_buffer_atomic_dec: + return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_DEC); - switch (IntrID) { - case Intrinsic::amdgcn_struct_buffer_atomic_swap: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SWAP; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_add: - Opcode = AMDGPUISD::BUFFER_ATOMIC_ADD; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_sub: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SUB; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_smin: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SMIN; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_umin: - Opcode = AMDGPUISD::BUFFER_ATOMIC_UMIN; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_smax: - Opcode = AMDGPUISD::BUFFER_ATOMIC_SMAX; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_umax: - Opcode = AMDGPUISD::BUFFER_ATOMIC_UMAX; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_and: - Opcode = AMDGPUISD::BUFFER_ATOMIC_AND; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_or: - Opcode = AMDGPUISD::BUFFER_ATOMIC_OR; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_xor: - Opcode = AMDGPUISD::BUFFER_ATOMIC_XOR; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_inc: - Opcode = AMDGPUISD::BUFFER_ATOMIC_INC; - break; - case Intrinsic::amdgcn_struct_buffer_atomic_dec: - Opcode = AMDGPUISD::BUFFER_ATOMIC_DEC; - break; - default: - llvm_unreachable("unhandled atomic opcode"); - } - - return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT, - M->getMemOperand()); - } case Intrinsic::amdgcn_buffer_atomic_cmpswap: { unsigned Slc = cast(Op.getOperand(7))->getZExtValue(); unsigned IdxEn = 1;