[AMDGPU] Convert rcp to rcp_iflag

[rampitec]

Summary

If a source of rcp instruction is a result of any conversion from
an integer convert it into rcp_iflag instruction. No FP exception
can ever happen except division by zero if a single precision rcp
argument is a representation of an integral number.

[arsenm]

lib/Target/AMDGPU/SIISelLowering.cpp

6641 ↗

(On Diff #152775)

Preserve fast math flags?

6644 ↗

(On Diff #152775)

This can just return SDValue(). The base PerformDAGCombine isn't going to do anything here

[rampitec]

lib/Target/AMDGPU/SIISelLowering.cpp

6641 ↗

(On Diff #152775)

Makes sense.

6644 ↗

(On Diff #152775)

In fact not, there is combining code in parent and having no this call results in tests regression.

[arsenm]

lib/Target/AMDGPU/SIISelLowering.cpp

6644 ↗

(On Diff #152775)

Oh, I see there is constant folding there already.

It would probably be better then to either factor that into a function which can be called from here, or to just handle all of this is one performRcpCombine

[arsenm]

LGTM

llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h

protected:
   SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
                              SDValue RHS, DAGCombinerInfo &DCI) const;
   SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
   static EVT getEquivalentMemType(LLVMContext &Context, EVT VT);
 
   virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
                                      SelectionDAG &DAG) const;
 
   /// Return 64-bit value Op as two 32-bit integers.
   std::pair<SDValue, SDValue> split64BitValue(SDValue Op,
enum NodeType : unsigned {
   TRIG_PREOP, // 1 ULP max error for f64
 
   // RCP, RSQ - For f32, 1 ULP max error, no denormal handling.
   //            For f64, max error 2^29 ULP, handles denormals.
   RCP,
   RSQ,
   RCP_LEGACY,
   RSQ_LEGACY,
+  RCP_IFLAG,
   FMUL_LEGACY,
   RSQ_CLAMP,
   LDEXP,
   FP_CLASS,
   DOT4,
   CARRY,
   BORROW,
   BFE_U32, // Extract range of bits with zero extension to 32-bits.

llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

static bool fnegFoldsIntoOp(unsigned Opc) {
   case ISD::FMINNUM:
   case ISD::FMAXNUM:
   case ISD::FSIN:
   case ISD::FTRUNC:
   case ISD::FRINT:
   case ISD::FNEARBYINT:
   case AMDGPUISD::RCP:
   case AMDGPUISD::RCP_LEGACY:
+  case AMDGPUISD::RCP_IFLAG:
   case AMDGPUISD::SIN_HW:
   case AMDGPUISD::FMUL_LEGACY:
   case AMDGPUISD::FMIN_LEGACY:
   case AMDGPUISD::FMAX_LEGACY:
     return true;
   default:
     return false;
   }
SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
   }
   case ISD::FP_EXTEND:
   case ISD::FTRUNC:
   case ISD::FRINT:
   case ISD::FNEARBYINT: // XXX - Should fround be handled?
   case ISD::FSIN:
   case AMDGPUISD::RCP:
   case AMDGPUISD::RCP_LEGACY:
+  case AMDGPUISD::RCP_IFLAG:
   case AMDGPUISD::SIN_HW: {
     SDValue CvtSrc = N0.getOperand(0);
     if (CvtSrc.getOpcode() == ISD::FNEG) {
       // (fneg (fp_extend (fneg x))) -> (fp_extend x)
       // (fneg (rcp (fneg x))) -> (rcp x)
       return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
     }
 
SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
                                   DAG.getConstant(0x7fff, SL, SrcVT));
     return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
   }
   default:
     return SDValue();
   }
 }
 
+SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
+                                                DAGCombinerInfo &DCI) const {
+  const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
+  if (!CFP)
+    return SDValue();
+
+  // XXX - Should this flush denormals?
+  const APFloat &Val = CFP->getValueAPF();
+  APFloat One(Val.getSemantics(), "1.0");
+  return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
+}
+
 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   switch(N->getOpcode()) {
   default:
     break;
case AMDGPUISD::BFE_U32: {
     }
 
     break;
   }
   case ISD::LOAD:
     return performLoadCombine(N, DCI);
   case ISD::STORE:
     return performStoreCombine(N, DCI);
-  case AMDGPUISD::RCP: {
-    if (const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
-      // XXX - Should this flush denormals?
-      const APFloat &Val = CFP->getValueAPF();
-      APFloat One(Val.getSemantics(), "1.0");
-      return DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
-    }
-
-    break;
-  }
+  case AMDGPUISD::RCP:
+  case AMDGPUISD::RCP_IFLAG:
+    return performRcpCombine(N, DCI);
   case ISD::AssertZext:
   case ISD::AssertSext:
     return performAssertSZExtCombine(N, DCI);
   }
   return SDValue();
 }
 
 //===----------------------------------------------------------------------===//
const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(DIV_FMAS)
   NODE_NAME_CASE(DIV_FIXUP)
   NODE_NAME_CASE(FMAD_FTZ)
   NODE_NAME_CASE(TRIG_PREOP)
   NODE_NAME_CASE(RCP)
   NODE_NAME_CASE(RSQ)
   NODE_NAME_CASE(RCP_LEGACY)
   NODE_NAME_CASE(RSQ_LEGACY)
+  NODE_NAME_CASE(RCP_IFLAG)
   NODE_NAME_CASE(FMUL_LEGACY)
   NODE_NAME_CASE(RSQ_CLAMP)
   NODE_NAME_CASE(LDEXP)
   NODE_NAME_CASE(FP_CLASS)
   NODE_NAME_CASE(DOT4)
   NODE_NAME_CASE(CARRY)
   NODE_NAME_CASE(BORROW)
   NODE_NAME_CASE(BFE_U32)

llvm/trunk/lib/Target/AMDGPU/AMDGPUInstrInfo.td

 
 // out = 1.0 / sqrt(a)
 def AMDGPUrsq : SDNode<"AMDGPUISD::RSQ", SDTFPUnaryOp>;
 
 // out = 1.0 / sqrt(a)
 def AMDGPUrcp_legacy : SDNode<"AMDGPUISD::RCP_LEGACY", SDTFPUnaryOp>;
 def AMDGPUrsq_legacy : SDNode<"AMDGPUISD::RSQ_LEGACY", SDTFPUnaryOp>;
 
+def AMDGPUrcp_iflag : SDNode<"AMDGPUISD::RCP_IFLAG", SDTFPUnaryOp>;
+
 // out = 1.0 / sqrt(a) result clamped to +/- max_float.
 def AMDGPUrsq_clamp : SDNode<"AMDGPUISD::RSQ_CLAMP", SDTFPUnaryOp>;
 
 def AMDGPUldexp : SDNode<"AMDGPUISD::LDEXP", AMDGPULdExpOp>;
 
 def AMDGPUpkrtz_f16_f32 : SDNode<"AMDGPUISD::CVT_PKRTZ_F16_F32", AMDGPUFPPackOp>;
 def AMDGPUpknorm_i16_f32 : SDNode<"AMDGPUISD::CVT_PKNORM_I16_F32", AMDGPUFPPackOp>;
 def AMDGPUpknorm_u16_f32 : SDNode<"AMDGPUISD::CVT_PKNORM_U16_F32", AMDGPUFPPackOp>;

llvm/trunk/lib/Target/AMDGPU/SIISelLowering.h

class SITargetLowering final : public AMDGPUTargetLowering {
   SDValue performAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performAddCarrySubCarryCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performClampCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
   bool isLegalFlatAddressingMode(const AddrMode &AM) const;
   bool isLegalGlobalAddressingMode(const AddrMode &AM) const;
   bool isLegalMUBUFAddressingMode(const AddrMode &AM) const;
 
   unsigned isCFIntrinsic(const SDNode *Intr) const;
 
   void createDebuggerPrologueStackObjects(MachineFunction &MF) const;

llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp

static bool fp16SrcZerosHighBits(unsigned Opc) {
   case AMDGPUISD::COS_HW:
   case AMDGPUISD::SIN_HW:
   case AMDGPUISD::FMIN3:
   case AMDGPUISD::FMAX3:
   case AMDGPUISD::FMED3:
   case AMDGPUISD::FMAD_FTZ:
   case AMDGPUISD::RCP:
   case AMDGPUISD::RSQ:
+  case AMDGPUISD::RCP_IFLAG:
   case AMDGPUISD::LDEXP:
     return true;
   default:
     // fcopysign, select and others may be lowered to 32-bit bit operations
     // which don't zero the high bits.
     return false;
   }
 }
SDValue SITargetLowering::performClassCombine(SDNode *N,
   }
 
   if (N->getOperand(0).isUndef())
     return DAG.getUNDEF(MVT::i1);
 
   return SDValue();
 }
 
+SDValue SITargetLowering::performRcpCombine(SDNode *N,
+                                            DAGCombinerInfo &DCI) const {
+  EVT VT = N->getValueType(0);
+  SDValue N0 = N->getOperand(0);
+
+  if (N0.isUndef())
+    return N0;
+
+  if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP ||
+                         N0.getOpcode() == ISD::SINT_TO_FP)) {
+    return DCI.DAG.getNode(AMDGPUISD::RCP_IFLAG, SDLoc(N), VT, N0,
+                           N->getFlags());
+  }
+
+  return AMDGPUTargetLowering::performRcpCombine(N, DCI);
+}
+
 static bool isKnownNeverSNan(SelectionDAG &DAG, SDValue Op) {
   if (!DAG.getTargetLoweringInfo().hasFloatingPointExceptions())
     return true;
 
   return DAG.isKnownNeverNaN(Op);
 }
 
 static bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::XOR:
     return performXorCombine(N, DCI);
   case ISD::ZERO_EXTEND:
     return performZeroExtendCombine(N, DCI);
   case AMDGPUISD::FP_CLASS:
     return performClassCombine(N, DCI);
   case ISD::FCANONICALIZE:
     return performFCanonicalizeCombine(N, DCI);
-  case AMDGPUISD::FRACT:
   case AMDGPUISD::RCP:
+    return performRcpCombine(N, DCI);
+  case AMDGPUISD::FRACT:
   case AMDGPUISD::RSQ:
   case AMDGPUISD::RCP_LEGACY:
   case AMDGPUISD::RSQ_LEGACY:
+  case AMDGPUISD::RCP_IFLAG:
   case AMDGPUISD::RSQ_CLAMP:
   case AMDGPUISD::LDEXP: {
     SDValue Src = N->getOperand(0);
     if (Src.isUndef())
       return Src;
     break;
   }
   case ISD::SINT_TO_FP:

llvm/trunk/lib/Target/AMDGPU/VOP1Instructions.td

 defm V_CEIL_F32 : VOP1Inst <"v_ceil_f32", VOP_F32_F32, fceil>;
 defm V_RNDNE_F32 : VOP1Inst <"v_rndne_f32", VOP_F32_F32, frint>;
 defm V_FLOOR_F32 : VOP1Inst <"v_floor_f32", VOP_F32_F32, ffloor>;
 
 let SchedRW = [WriteQuarterRate32] in {
 defm V_EXP_F32 : VOP1Inst <"v_exp_f32", VOP_F32_F32, fexp2>;
 defm V_LOG_F32 : VOP1Inst <"v_log_f32", VOP_F32_F32, flog2>;
 defm V_RCP_F32 : VOP1Inst <"v_rcp_f32", VOP_F32_F32, AMDGPUrcp>;
-defm V_RCP_IFLAG_F32 : VOP1Inst <"v_rcp_iflag_f32", VOP_F32_F32>;
+defm V_RCP_IFLAG_F32 : VOP1Inst <"v_rcp_iflag_f32", VOP_F32_F32, AMDGPUrcp_iflag>;
 defm V_RSQ_F32 : VOP1Inst <"v_rsq_f32", VOP_F32_F32, AMDGPUrsq>;
 defm V_SQRT_F32 : VOP1Inst <"v_sqrt_f32", VOP_F32_F32, fsqrt>;
 } // End SchedRW = [WriteQuarterRate32]
 
 let SchedRW = [WriteDouble] in {
 defm V_RCP_F64 : VOP1Inst <"v_rcp_f64", VOP_F64_F64, AMDGPUrcp>;
 defm V_RSQ_F64 : VOP1Inst <"v_rsq_f64", VOP_F64_F64, AMDGPUrsq>;
 } // End SchedRW = [WriteDouble];

llvm/trunk/test/CodeGen/AMDGPU/rcp_iflag.ll

+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}rcp_uint:
+; GCN: v_rcp_iflag_f32_e32
+define amdgpu_kernel void @rcp_uint(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+  %load = load i32, i32 addrspace(1)* %in, align 4
+  %cvt = uitofp i32 %load to float
+  %div = fdiv float 1.000000e+00, %cvt
+  store float %div, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}rcp_sint:
+; GCN: v_rcp_iflag_f32_e32
+define amdgpu_kernel void @rcp_sint(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+  %load = load i32, i32 addrspace(1)* %in, align 4
+  %cvt = sitofp i32 %load to float
+  %div = fdiv float 1.000000e+00, %cvt
+  store float %div, float addrspace(1)* %out, align 4
+  ret void
+}

llvm/trunk/test/CodeGen/AMDGPU/sdiv.ll

 define amdgpu_kernel void @sdiv_v4i32_4(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
   %num = load <4 x i32>, <4 x i32> addrspace(1) * %in
   %result = sdiv <4 x i32> %num, <i32 4, i32 4, i32 4, i32 4>
   store <4 x i32> %result, <4 x i32> addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_sdiv_i8:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_bfe_i32 [[BFE:v[0-9]+]], v{{[0-9]+}}, 0, 8
 ; SI: buffer_store_dword [[BFE]]
 define amdgpu_kernel void @v_sdiv_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
   %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
   %num = load i8, i8 addrspace(1) * %in
   %den = load i8, i8 addrspace(1) * %den_ptr
   %result = sdiv i8 %num, %den
   %result.ext = sext i8 %result to i32
   store i32 %result.ext, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_sdiv_i23:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_bfe_i32 [[BFE:v[0-9]+]], v{{[0-9]+}}, 0, 23
 ; SI: buffer_store_dword [[BFE]]
 define amdgpu_kernel void @v_sdiv_i23(i32 addrspace(1)* %out, i23 addrspace(1)* %in) {
   %den_ptr = getelementptr i23, i23 addrspace(1)* %in, i23 1
   %num = load i23, i23 addrspace(1) * %in
   %den = load i23, i23 addrspace(1) * %den_ptr
   %result = sdiv i23 %num, %den
   %result.ext = sext i23 %result to i32
   store i32 %result.ext, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_sdiv_i24:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_bfe_i32 [[BFE:v[0-9]+]], v{{[0-9]+}}, 0, 24
 ; SI: buffer_store_dword [[BFE]]
 define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)* %in) {
   %den_ptr = getelementptr i24, i24 addrspace(1)* %in, i24 1
   %num = load i24, i24 addrspace(1) * %in
   %den = load i24, i24 addrspace(1) * %den_ptr
   %result = sdiv i24 %num, %den
   %result.ext = sext i24 %result to i32

llvm/trunk/test/CodeGen/AMDGPU/sdivrem24.ll

 ; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}sdiv24_i8:
 ; SI: v_cvt_f32_i32
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_i32_f32
 
 ; EG: INT_TO_FLT
 ; EG-DAG: INT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_INT
 define amdgpu_kernel void @sdiv24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) {
   %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
   %num = load i8, i8 addrspace(1) * %in
   %den = load i8, i8 addrspace(1) * %den_ptr
   %result = sdiv i8 %num, %den
   store i8 %result, i8 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}sdiv24_i16:
 ; SI: v_cvt_f32_i32
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_i32_f32
 
 ; EG: INT_TO_FLT
 ; EG-DAG: INT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_INT
 define amdgpu_kernel void @sdiv24_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
   %den_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
   %num = load i16, i16 addrspace(1) * %in, align 2
   %den = load i16, i16 addrspace(1) * %den_ptr, align 2
   %result = sdiv i16 %num, %den
   store i16 %result, i16 addrspace(1)* %out, align 2
   ret void
 }
 
 ; FUNC-LABEL: {{^}}sdiv24_i32:
 ; SI: v_cvt_f32_i32
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_i32_f32
 
 ; EG: INT_TO_FLT
 ; EG-DAG: INT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_INT
 define amdgpu_kernel void @sdiv24_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
define amdgpu_kernel void @test_no_sdiv24_i32_2(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %result = sdiv i32 %num.i24, %den.i24
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem24_i8:
 ; SI: v_cvt_f32_i32
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_i32_f32
 
 ; EG: INT_TO_FLT
 ; EG-DAG: INT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_INT
 define amdgpu_kernel void @srem24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) {
   %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
   %num = load i8, i8 addrspace(1) * %in
   %den = load i8, i8 addrspace(1) * %den_ptr
   %result = srem i8 %num, %den
   store i8 %result, i8 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem24_i16:
 ; SI: v_cvt_f32_i32
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_i32_f32
 
 ; EG: INT_TO_FLT
 ; EG-DAG: INT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_INT
 define amdgpu_kernel void @srem24_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
   %den_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
   %num = load i16, i16 addrspace(1) * %in, align 2
   %den = load i16, i16 addrspace(1) * %den_ptr, align 2
   %result = srem i16 %num, %den
   store i16 %result, i16 addrspace(1)* %out, align 2
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem24_i32:
 ; SI: v_cvt_f32_i32
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_i32_f32
 
 ; EG: INT_TO_FLT
 ; EG-DAG: INT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_INT
 define amdgpu_kernel void @srem24_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
define amdgpu_kernel void @no_srem25_i25_i24_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den.i24 = ashr i32 %den.i24.0, 8
   %result = srem i32 %num.i25, %den.i24
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem25_i24_i11_i32:
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24
 
 ; EG: INT_TO_FLT
 ; EG: RECIP_IEEE
 define amdgpu_kernel void @srem25_i24_i11_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %num = load i32, i32 addrspace(1) * %in, align 4
   %den = load i32, i32 addrspace(1) * %den_ptr, align 4
   %num.i24.0 = shl i32 %num, 8
   %den.i11.0 = shl i32 %den, 21
   %num.i24 = ashr i32 %num.i24.0, 8
   %den.i11 = ashr i32 %den.i11.0, 21
   %result = srem i32 %num.i24, %den.i11
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem25_i11_i24_i32:
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24
 
 ; EG: INT_TO_FLT
 ; EG: RECIP_IEEE
 define amdgpu_kernel void @srem25_i11_i24_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %num = load i32, i32 addrspace(1) * %in, align 4
   %den = load i32, i32 addrspace(1) * %den_ptr, align 4
   %num.i11.0 = shl i32 %num, 21
   %den.i24.0 = shl i32 %den, 8
   %num.i11 = ashr i32 %num.i11.0, 21
   %den.i24 = ashr i32 %den.i24.0, 8
   %result = srem i32 %num.i11, %den.i24
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem25_i17_i12_i32:
 ; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 17
 
 ; EG: INT_TO_FLT
 ; EG: RECIP_IEEE
 define amdgpu_kernel void @srem25_i17_i12_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %num = load i32, i32 addrspace(1) * %in, align 4
   %den = load i32, i32 addrspace(1) * %den_ptr, align 4
   %num.i17.0 = shl i32 %num, 15
   %den.i12.0 = shl i32 %den, 20
   %num.i17 = ashr i32 %num.i17.0, 15
   %den.i12 = ashr i32 %den.i12.0, 20
   %result = sdiv i32 %num.i17, %den.i12
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }

llvm/trunk/test/CodeGen/AMDGPU/udiv.ll

define amdgpu_kernel void @udiv_i32_div_k_odd(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %a = load i32, i32 addrspace(1)* %in
   %result = udiv i32 %a, 34259183
   store i32 %result, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_udiv_i8:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0xff, v{{[0-9]+}}
 ; SI: buffer_store_dword [[TRUNC]]
 define amdgpu_kernel void @v_udiv_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
   %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
   %num = load i8, i8 addrspace(1) * %in
   %den = load i8, i8 addrspace(1) * %den_ptr
   %result = udiv i8 %num, %den
   %result.ext = zext i8 %result to i32
   store i32 %result.ext, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_udiv_i16:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0xffff, v{{[0-9]+}}
 ; SI: buffer_store_dword [[TRUNC]]
 define amdgpu_kernel void @v_udiv_i16(i32 addrspace(1)* %out, i16 addrspace(1)* %in) {
   %den_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
   %num = load i16, i16 addrspace(1) * %in
   %den = load i16, i16 addrspace(1) * %den_ptr
   %result = udiv i16 %num, %den
   %result.ext = zext i16 %result to i32
   store i32 %result.ext, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_udiv_i23:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0x7fffff, v{{[0-9]+}}
 ; SI: buffer_store_dword [[TRUNC]]
 define amdgpu_kernel void @v_udiv_i23(i32 addrspace(1)* %out, i23 addrspace(1)* %in) {
   %den_ptr = getelementptr i23, i23 addrspace(1)* %in, i23 1
   %num = load i23, i23 addrspace(1) * %in
   %den = load i23, i23 addrspace(1) * %den_ptr
   %result = udiv i23 %num, %den
   %result.ext = zext i23 %result to i32

llvm/trunk/test/CodeGen/AMDGPU/udivrem24.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}udiv24_i8:
 ; SI: v_cvt_f32_ubyte
 ; SI: v_cvt_f32_ubyte
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_u32_f32
 
 ; EG: UINT_TO_FLT
 ; EG-DAG: UINT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_UINT
 define amdgpu_kernel void @udiv24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) {
   %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
   %num = load i8, i8 addrspace(1) * %in
   %den = load i8, i8 addrspace(1) * %den_ptr
   %result = udiv i8 %num, %den
   store i8 %result, i8 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}udiv24_i16:
 ; SI: v_cvt_f32_u32
 ; SI: v_cvt_f32_u32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_u32_f32
 
 ; EG: UINT_TO_FLT
 ; EG-DAG: UINT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_UINT
 define amdgpu_kernel void @udiv24_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
   %den_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
   %num = load i16, i16 addrspace(1) * %in, align 2
   %den = load i16, i16 addrspace(1) * %den_ptr, align 2
   %result = udiv i16 %num, %den
   store i16 %result, i16 addrspace(1)* %out, align 2
   ret void
 }
 
 ; FUNC-LABEL: {{^}}udiv23_i32:
 ; SI: v_cvt_f32_u32
 ; SI-DAG: v_cvt_f32_u32
-; SI-DAG: v_rcp_f32
+; SI-DAG: v_rcp_iflag_f32
 ; SI: v_cvt_u32_f32
 
 ; EG: UINT_TO_FLT
 ; EG-DAG: UINT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_UINT
 define amdgpu_kernel void @udiv23_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
define amdgpu_kernel void @test_no_udiv24_i32_2(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %result = udiv i32 %num.i24, %den.i24
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}urem24_i8:
 ; SI: v_cvt_f32_ubyte
 ; SI: v_cvt_f32_ubyte
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_u32_f32
 
 ; EG: UINT_TO_FLT
 ; EG-DAG: UINT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_UINT
 define amdgpu_kernel void @urem24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) {
   %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
   %num = load i8, i8 addrspace(1) * %in
   %den = load i8, i8 addrspace(1) * %den_ptr
   %result = urem i8 %num, %den
   store i8 %result, i8 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}urem24_i16:
 ; SI: v_cvt_f32_u32
 ; SI: v_cvt_f32_u32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
 ; SI: v_cvt_u32_f32
 
 ; EG: UINT_TO_FLT
 ; EG-DAG: UINT_TO_FLT
 ; EG-DAG: RECIP_IEEE
 ; EG: FLT_TO_UINT
 define amdgpu_kernel void @urem24_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
   %den_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
define amdgpu_kernel void @test_no_urem24_i32_2(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %num.i24 = lshr i32 %num.i24.0, 7
   %den.i24 = lshr i32 %den.i24.0, 8
   %result = urem i32 %num.i24, %den.i24
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}test_udiv24_u16_u23_i32:
-; SI-DAG: v_rcp_f32
+; SI-DAG: v_rcp_iflag_f32
 ; SI-DAG: s_mov_b32 [[MASK:s[0-9]+]], 0x7fffff{{$}}
 ; SI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]],
 
 ; EG: RECIP_IEEE
 define amdgpu_kernel void @test_udiv24_u16_u23_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %num = load i32, i32 addrspace(1) * %in, align 4
   %den = load i32, i32 addrspace(1) * %den_ptr, align 4
   %num.i16.0 = shl i32 %num, 16
   %den.i23.0 = shl i32 %den, 9
   %num.i16 = lshr i32 %num.i16.0, 16
   %den.i23 = lshr i32 %den.i23.0, 9
   %result = udiv i32 %num.i16, %den.i23
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}test_udiv24_u23_u16_i32:
-; SI-DAG: v_rcp_f32
+; SI-DAG: v_rcp_iflag_f32
 ; SI-DAG: s_mov_b32 [[MASK:s[0-9]+]], 0x7fffff{{$}}
 ; SI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]],
 
 ; EG: RECIP_IEEE
 define amdgpu_kernel void @test_udiv24_u23_u16_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %num = load i32, i32 addrspace(1) * %in, align 4
   %den = load i32, i32 addrspace(1) * %den_ptr, align 4
   %num.i23.0 = shl i32 %num, 9
   %den.i16.0 = shl i32 %den, 16
   %num.i23 = lshr i32 %num.i23.0, 9
   %den.i16 = lshr i32 %den.i16.0, 16
   %result = udiv i32 %num.i23, %den.i16
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }