AMDGPU: Use more accurate fast f64 fdiv

[arsenm]

Summary

A raw v_rcp_f64 isn't accurate enough, so start applying correction.

[rampitec]

LGTM. Please also check with Brian correction sequence.

[b-sumner]

That sequence will give a very accurate result as long as overflow and underflow is avoided. LGTM.

[arsenm]

2a0db8d70eeb0c4c09e4c91b365630eefbbf3993

[foad]

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

3115

Can you add a comment here and/or in the sdag equivalent showing what the code you're building here will look like, and preferably where it came from and what kind of accuracy you expect from it?

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

 //
 //   1/x -> fdiv.fast(1,x)  when !fpmath >= 2.5ulp.
 //
 // NOTE: rcp is the preference in cases that both are legal.
 bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) {
 
   Type *Ty = FDiv.getType()->getScalarType();
 
+  // The f64 rcp/rsq approximations are pretty inaccurate. We can do an
+  // expansion around them in codegen.
+  if (Ty->isDoubleTy())
+    return false;
+
   // No intrinsic for fdiv16 if target does not support f16.
   if (Ty->isHalfTy() && !ST->has16BitInsts())
     return false;
 
   const FPMathOperator *FPOp = cast<const FPMathOperator>(&FDiv);
   const float ReqdAccuracy =  FPOp->getFPAccuracy();
 
   // Inaccurate rcp is allowed with unsafe-fp-math or afn.

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h

public:
   bool legalizeFDIV16(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &B) const;
   bool legalizeFDIV32(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &B) const;
   bool legalizeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &B) const;
   bool legalizeFastUnsafeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI,
                               MachineIRBuilder &B) const;
+  bool legalizeFastUnsafeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                MachineIRBuilder &B) const;
   bool legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI,
                               MachineIRBuilder &B) const;
 
   bool legalizeRsqClampIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
                                  MachineIRBuilder &B) const;
 
   bool legalizeDSAtomicFPIntrinsic(LegalizerHelper &Helper,
                                    MachineInstr &MI, Intrinsic::ID IID) const;

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
                                        MachineRegisterInfo &MRI,
                                        MachineIRBuilder &B) const {
   Register Dst = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(Dst);
   LLT S16 = LLT::scalar(16);
   LLT S32 = LLT::scalar(32);
   LLT S64 = LLT::scalar(64);
 
-  if (legalizeFastUnsafeFDIV(MI, MRI, B))
-    return true;
-
   if (DstTy == S16)
     return legalizeFDIV16(MI, MRI, B);
   if (DstTy == S32)
     return legalizeFDIV32(MI, MRI, B);
   if (DstTy == S64)
     return legalizeFDIV64(MI, MRI, B);
 
   return false;
auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy}, false)
     .addUse(RHS)
     .setMIFlags(Flags);
   B.buildFMul(Res, LHS, RCP, Flags);
 
   MI.eraseFromParent();
   return true;
 }
 
+bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV64(MachineInstr &MI,
+                                                   MachineRegisterInfo &MRI,
+                                                   MachineIRBuilder &B) const {
+  Register Res = MI.getOperand(0).getReg();
+  Register X = MI.getOperand(1).getReg();
+  Register Y = MI.getOperand(2).getReg();
+  uint16_t Flags = MI.getFlags();
+  LLT ResTy = MRI.getType(Res);
+
+  const MachineFunction &MF = B.getMF();
+  bool AllowInaccurateRcp = MF.getTarget().Options.UnsafeFPMath ||
+                            MI.getFlag(MachineInstr::FmAfn);
+
+  if (!AllowInaccurateRcp)
+    return false;
+
+  auto NegY = B.buildFNeg(ResTy, Y);
+  auto One = B.buildFConstant(ResTy, 1.0);
+
+  auto R = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy}, false)
+    .addUse(Y)
+    .setMIFlags(Flags);
+
+  auto Tmp0 = B.buildFMA(ResTy, NegY, R, One);
+  R = B.buildFMA(ResTy, Tmp0, R, R);
+
+  auto Tmp1 = B.buildFMA(ResTy, NegY, R, One);
+  R = B.buildFMA(ResTy, Tmp1, R, R);
+
+  auto Ret = B.buildFMul(ResTy, X, R);
+  auto Tmp2 = B.buildFMA(ResTy, NegY, Ret, X);
+
+  B.buildFMA(Res, Tmp2, R, Ret);
+  MI.eraseFromParent();
+  return true;
+}
+
 bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI,
                                          MachineRegisterInfo &MRI,
                                          MachineIRBuilder &B) const {
+  if (legalizeFastUnsafeFDIV(MI, MRI, B))
+    return true;
+
   Register Res = MI.getOperand(0).getReg();
   Register LHS = MI.getOperand(1).getReg();
   Register RHS = MI.getOperand(2).getReg();
 
   uint16_t Flags = MI.getFlags();
 
   LLT S16 = LLT::scalar(16);
   LLT S32 = LLT::scalar(32);
B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
       .addImm(SPDenormMode)
       .addImm(SPDenormModeBitField);
   }
 }
 
 bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
                                          MachineRegisterInfo &MRI,
                                          MachineIRBuilder &B) const {
+  if (legalizeFastUnsafeFDIV(MI, MRI, B))
+    return true;
+
   Register Res = MI.getOperand(0).getReg();
   Register LHS = MI.getOperand(1).getReg();
   Register RHS = MI.getOperand(2).getReg();
   const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
   AMDGPU::SIModeRegisterDefaults Mode = MFI->getMode();
 
   uint16_t Flags = MI.getFlags();
 
bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
 
   MI.eraseFromParent();
   return true;
 }
 
 bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI,
                                          MachineRegisterInfo &MRI,
                                          MachineIRBuilder &B) const {
+  if (legalizeFastUnsafeFDIV64(MI, MRI, B))
+    return true;
+
   Register Res = MI.getOperand(0).getReg();
   Register LHS = MI.getOperand(1).getReg();
   Register RHS = MI.getOperand(2).getReg();
 
   uint16_t Flags = MI.getFlags();
 
   LLT S64 = LLT::scalar(64);
   LLT S1 = LLT::scalar(1);

llvm/lib/Target/AMDGPU/SIISelLowering.h

private:
   // offset and figures out how to split it between voffset and immoffset.
   std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
                                                  SelectionDAG &DAG) const;
 
   SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
   SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFastUnsafeFDIV64(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV16(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
   }
 
   // Turn into multiply by the reciprocal.
   // x / y -> x * (1.0 / y)
   SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
   return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags);
 }
 
+SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  SDLoc SL(Op);
+  SDValue X = Op.getOperand(0);
+  SDValue Y = Op.getOperand(1);
+  EVT VT = Op.getValueType();
+  const SDNodeFlags Flags = Op->getFlags();
+
+  bool AllowInaccurateDiv = Flags.hasApproximateFuncs() ||
+                            DAG.getTarget().Options.UnsafeFPMath;
+  if (!AllowInaccurateDiv)
+    return SDValue();
+
+  SDValue NegY = DAG.getNode(ISD::FNEG, SL, VT, Y);
+  SDValue One = DAG.getConstantFP(1.0, SL, VT);
+
+  SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y);
+  SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
+
+  R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R);
+  SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
+  R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R);
+  SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R);
+  SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X);
+  return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret);
+}
+
 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
                           EVT VT, SDValue A, SDValue B, SDValue GlueChain,
                           SDNodeFlags Flags) {
   if (GlueChain->getNumValues() <= 1) {
     return DAG.getNode(Opcode, SL, VT, A, B, Flags);
   }
 
   assert(GlueChain->getNumValues() == 3);
SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
   SDValue Scale = NumeratorScaled.getValue(1);
   SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32,
                              {Fma4, Fma1, Fma3, Scale}, Flags);
 
   return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags);
 }
 
 SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
-  if (DAG.getTarget().Options.UnsafeFPMath)
-    return lowerFastUnsafeFDIV(Op, DAG);
+  if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
+    return FastLowered;
 
   SDLoc SL(Op);
   SDValue X = Op.getOperand(0);
   SDValue Y = Op.getOperand(1);
 
   const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
 
   SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1);

llvm/lib/Target/AMDGPU/SIInstructions.td

 >;
 
 //===----------------------------------------------------------------------===//
 // VOP1 Patterns
 //===----------------------------------------------------------------------===//
 
 let OtherPredicates = [UnsafeFPMath] in {
 
-//def : RcpPat<V_RCP_F64_e32, f64>;
 //defm : RsqPat<V_RSQ_F64_e32, f64>;
 //defm : RsqPat<V_RSQ_F32_e32, f32>;
 
 def : RsqPat<V_RSQ_F32_e32, f32>;
 def : RsqPat<V_RSQ_F64_e32, f64>;
 
 // Convert (x - floor(x)) to fract(x)
 def : GCNPat <

llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll

; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv double %a, %b
   ret double %fdiv
 }
 
 define double @v_fdiv_f64_afn(double %a, double %b) {
 ; GCN-LABEL: v_fdiv_f64_afn:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[2:3]
-; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GCN-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GCN-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[2:3], v[6:7], v[0:1]
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[4:5], v[6:7]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv afn double %a, %b
   ret double %fdiv
 }
 
 define double @v_fdiv_f64_ulp25(double %a, double %b) {
 ; GFX6-LABEL: v_fdiv_f64_ulp25:
 ; GFX6:       ; %bb.0:
; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv arcp double 1.0, %x
   ret double %fdiv
 }
 
 define double @v_rcp_f64_arcp_afn(double %x) {
 ; GCN-LABEL: v_rcp_f64_arcp_afn:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[0:1], v[0:1]
+; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[0:1]
+; GCN-NEXT:    v_fma_f64 v[4:5], -v[0:1], v[2:3], 1.0
+; GCN-NEXT:    v_fma_f64 v[2:3], v[4:5], v[2:3], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[4:5], -v[0:1], v[2:3], 1.0
+; GCN-NEXT:    v_fma_f64 v[2:3], v[4:5], v[2:3], v[2:3]
+; GCN-NEXT:    v_mul_f64 v[4:5], 1.0, v[2:3]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv arcp afn double 1.0, %x
   ret double %fdiv
 }
 
 define double @v_rcp_f64_ulp25(double %x) {
 ; GFX6-LABEL: v_rcp_f64_ulp25:
 ; GFX6:       ; %bb.0:
; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv double 1.0, %x, !fpmath !0
   ret double %fdiv
 }
 
 define double @v_fdiv_f64_afn_ulp25(double %a, double %b) {
 ; GCN-LABEL: v_fdiv_f64_afn_ulp25:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[2:3]
-; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GCN-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GCN-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[2:3], v[6:7], v[0:1]
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[4:5], v[6:7]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv afn double %a, %b, !fpmath !0
   ret double %fdiv
 }
 
 define double @v_fdiv_f64_arcp_ulp25(double %a, double %b) {
 ; GFX6-LABEL: v_fdiv_f64_arcp_ulp25:
 ; GFX6:       ; %bb.0:
; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv <2 x double> %a, %b
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_fdiv_v2f64_afn(<2 x double> %a, <2 x double> %b) {
 ; GCN-LABEL: v_fdiv_v2f64_afn:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[4:5]
-; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[6:7]
-; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
-; GCN-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GCN-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GCN-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GCN-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[10:11], v[14:15], v[10:11], v[10:11]
+; GCN-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[10:11], v[14:15], v[10:11], v[10:11]
+; GCN-NEXT:    v_mul_f64 v[12:13], v[0:1], v[8:9]
+; GCN-NEXT:    v_mul_f64 v[14:15], v[2:3], v[10:11]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[12:13], v[0:1]
+; GCN-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[14:15], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[12:13]
+; GCN-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[14:15]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv afn <2 x double> %a, %b
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_fdiv_v2f64_ulp25(<2 x double> %a, <2 x double> %b) {
 ; GFX6-LABEL: v_fdiv_v2f64_ulp25:
 ; GFX6:       ; %bb.0:
; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv arcp <2 x double> <double 1.0, double 1.0>, %x
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_rcp_v2f64_arcp_afn(<2 x double> %x) {
 ; GCN-LABEL: v_rcp_v2f64_arcp_afn:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[0:1], v[0:1]
-; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[2:3]
+; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[0:1]
+; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[8:9], -v[0:1], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[10:11], -v[2:3], v[6:7], 1.0
+; GCN-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[4:5]
+; GCN-NEXT:    v_fma_f64 v[6:7], v[10:11], v[6:7], v[6:7]
+; GCN-NEXT:    v_fma_f64 v[8:9], -v[0:1], v[4:5], 1.0
+; GCN-NEXT:    v_fma_f64 v[10:11], -v[2:3], v[6:7], 1.0
+; GCN-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[4:5]
+; GCN-NEXT:    v_fma_f64 v[6:7], v[10:11], v[6:7], v[6:7]
+; GCN-NEXT:    v_mul_f64 v[8:9], 1.0, v[4:5]
+; GCN-NEXT:    v_mul_f64 v[10:11], 1.0, v[6:7]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[4:5], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[10:11]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv arcp afn <2 x double> <double 1.0, double 1.0>, %x
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_rcp_v2f64_ulp25(<2 x double> %x) {
 ; GFX6-LABEL: v_rcp_v2f64_ulp25:
 ; GFX6:       ; %bb.0:
; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv <2 x double> <double 1.0, double 1.0>, %x, !fpmath !0
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_fdiv_v2f64_afn_ulp25(<2 x double> %a, <2 x double> %b) {
 ; GCN-LABEL: v_fdiv_v2f64_afn_ulp25:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[4:5]
-; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[6:7]
-; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
-; GCN-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GCN-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GCN-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GCN-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[10:11], v[14:15], v[10:11], v[10:11]
+; GCN-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[10:11], v[14:15], v[10:11], v[10:11]
+; GCN-NEXT:    v_mul_f64 v[12:13], v[0:1], v[8:9]
+; GCN-NEXT:    v_mul_f64 v[14:15], v[2:3], v[10:11]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[12:13], v[0:1]
+; GCN-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[14:15], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[12:13]
+; GCN-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[14:15]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv afn <2 x double> %a, %b, !fpmath !0
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_fdiv_v2f64_arcp_ulp25(<2 x double> %a, <2 x double> %b) {
 ; GFX6-LABEL: v_fdiv_v2f64_arcp_ulp25:
 ; GFX6:       ; %bb.0:
; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv arcp <2 x double> %a, %b, !fpmath !0
   ret <2 x double> %fdiv
 }
 
 define <2 x double> @v_fdiv_v2f64_arcp_afn_ulp25(<2 x double> %a, <2 x double> %b) {
 ; GCN-LABEL: v_fdiv_v2f64_arcp_afn_ulp25:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[4:5]
-; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[6:7]
-; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
-; GCN-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GCN-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GCN-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GCN-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[10:11], v[14:15], v[10:11], v[10:11]
+; GCN-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GCN-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GCN-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[8:9]
+; GCN-NEXT:    v_fma_f64 v[10:11], v[14:15], v[10:11], v[10:11]
+; GCN-NEXT:    v_mul_f64 v[12:13], v[0:1], v[8:9]
+; GCN-NEXT:    v_mul_f64 v[14:15], v[2:3], v[10:11]
+; GCN-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[12:13], v[0:1]
+; GCN-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[14:15], v[2:3]
+; GCN-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[12:13]
+; GCN-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[14:15]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fdiv = fdiv afn arcp <2 x double> %a, %b, !fpmath !0
   ret <2 x double> %fdiv
 }
 
 !0 = !{float 2.500000e+00}

llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll

 ; CI-LABEL: fast_frem_f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
+; CI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; CI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; CI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; CI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
-; CI-NEXT:    v_mul_f64 v[0:1], s[0:1], v[0:1]
+; CI-NEXT:    v_mul_f64 v[4:5], s[0:1], v[0:1]
+; CI-NEXT:    v_fma_f64 v[6:7], -s[2:3], v[4:5], v[2:3]
+; CI-NEXT:    v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
 ; CI-NEXT:    v_trunc_f64_e32 v[0:1], v[0:1]
 ; CI-NEXT:    v_fma_f64 v[0:1], -v[0:1], s[2:3], v[2:3]
 ; CI-NEXT:    v_mov_b32_e32 v2, s4
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fast_frem_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
+; VI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
-; VI-NEXT:    v_mul_f64 v[0:1], s[0:1], v[0:1]
+; VI-NEXT:    v_mul_f64 v[4:5], s[0:1], v[0:1]
+; VI-NEXT:    v_fma_f64 v[6:7], -s[2:3], v[4:5], v[2:3]
+; VI-NEXT:    v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
 ; VI-NEXT:    v_trunc_f64_e32 v[0:1], v[0:1]
 ; VI-NEXT:    v_fma_f64 v[0:1], -v[0:1], s[2:3], v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
    %r0 = load double, double addrspace(1)* %in1, align 8
    %r1 = load double, double addrspace(1)* %in2, align 8
    %r2 = frem fast double %r0, %r1
    store double %r2, double addrspace(1)* %out, align 8
    ret void
 }
 
 define amdgpu_kernel void @unsafe_frem_f64(double addrspace(1)* %out, double addrspace(1)* %in1,
 ; CI-LABEL: unsafe_frem_f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
+; CI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; CI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; CI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; CI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
-; CI-NEXT:    v_mul_f64 v[0:1], s[0:1], v[0:1]
+; CI-NEXT:    v_mul_f64 v[4:5], s[0:1], v[0:1]
+; CI-NEXT:    v_fma_f64 v[6:7], -s[2:3], v[4:5], v[2:3]
+; CI-NEXT:    v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
 ; CI-NEXT:    v_trunc_f64_e32 v[0:1], v[0:1]
 ; CI-NEXT:    v_fma_f64 v[0:1], -v[0:1], s[2:3], v[2:3]
 ; CI-NEXT:    v_mov_b32_e32 v2, s4
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: unsafe_frem_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
+; VI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
-; VI-NEXT:    v_mul_f64 v[0:1], s[0:1], v[0:1]
+; VI-NEXT:    v_mul_f64 v[4:5], s[0:1], v[0:1]
+; VI-NEXT:    v_fma_f64 v[6:7], -s[2:3], v[4:5], v[2:3]
+; VI-NEXT:    v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
 ; VI-NEXT:    v_trunc_f64_e32 v[0:1], v[0:1]
 ; VI-NEXT:    v_fma_f64 v[0:1], -v[0:1], s[2:3], v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
                              double addrspace(1)* %in2) #1 {
    %r0 = load double, double addrspace(1)* %in1, align 8

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir

bb.0:
     ; GFX9: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INT3]], [[FMA3]]
     ; GFX9: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[INT3]]
     ; GFX9: [[INT5:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fmas), [[FMA4]](s64), [[FMA3]](s64), [[FMUL]](s64), [[INT4]](s1)
     ; GFX9: [[INT6:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[INT5]](s64), [[COPY1]](s64), [[COPY]](s64)
     ; GFX9: $vgpr0_vgpr1 = COPY [[INT6]](s64)
     ; GFX9-UNSAFE-LABEL: name: test_fdiv_s64
     ; GFX9-UNSAFE: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9-UNSAFE: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9-UNSAFE: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY1]]
+    ; GFX9-UNSAFE: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX9-UNSAFE: [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[COPY1]](s64)
-    ; GFX9-UNSAFE: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[INT]]
-    ; GFX9-UNSAFE: $vgpr0_vgpr1 = COPY [[FMUL]](s64)
+    ; GFX9-UNSAFE: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT]], [[C]]
+    ; GFX9-UNSAFE: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FMA]], [[INT]], [[INT]]
+    ; GFX9-UNSAFE: [[FMA2:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMA1]], [[C]]
+    ; GFX9-UNSAFE: [[FMA3:%[0-9]+]]:_(s64) = G_FMA [[FMA2]], [[FMA1]], [[FMA1]]
+    ; GFX9-UNSAFE: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[FMA3]]
+    ; GFX9-UNSAFE: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[COPY]]
+    ; GFX9-UNSAFE: [[FMA5:%[0-9]+]]:_(s64) = G_FMA [[FMA4]], [[FMA3]], [[FMUL]]
+    ; GFX9-UNSAFE: $vgpr0_vgpr1 = COPY [[FMA5]](s64)
     ; GFX10-LABEL: name: test_fdiv_s64
     ; GFX10: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
     ; GFX10: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX10: [[INT:%[0-9]+]]:_(s64), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[COPY]](s64), [[COPY1]](s64), 0
     ; GFX10: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[INT]]
     ; GFX10: [[INT2:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[INT]](s64)
     ; GFX10: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT2]], [[C]]
bb.0:
     ; GFX9: [[INT13:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[INT12]](s64), [[UV3]](s64), [[UV1]](s64)
     ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[INT6]](s64), [[INT13]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-UNSAFE-LABEL: name: test_fdiv_v2s64
     ; GFX9-UNSAFE: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9-UNSAFE: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9-UNSAFE: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9-UNSAFE: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; GFX9-UNSAFE: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV2]]
+    ; GFX9-UNSAFE: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX9-UNSAFE: [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[UV2]](s64)
-    ; GFX9-UNSAFE: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[UV]], [[INT]]
+    ; GFX9-UNSAFE: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT]], [[C]]
+    ; GFX9-UNSAFE: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FMA]], [[INT]], [[INT]]
+    ; GFX9-UNSAFE: [[FMA2:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMA1]], [[C]]
+    ; GFX9-UNSAFE: [[FMA3:%[0-9]+]]:_(s64) = G_FMA [[FMA2]], [[FMA1]], [[FMA1]]
+    ; GFX9-UNSAFE: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[UV]], [[FMA3]]
+    ; GFX9-UNSAFE: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[UV]]
+    ; GFX9-UNSAFE: [[FMA5:%[0-9]+]]:_(s64) = G_FMA [[FMA4]], [[FMA3]], [[FMUL]]
+    ; GFX9-UNSAFE: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV3]]
     ; GFX9-UNSAFE: [[INT1:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[UV3]](s64)
-    ; GFX9-UNSAFE: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[UV1]], [[INT1]]
-    ; GFX9-UNSAFE: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMUL]](s64), [[FMUL1]](s64)
+    ; GFX9-UNSAFE: [[FMA6:%[0-9]+]]:_(s64) = G_FMA [[FNEG1]], [[INT1]], [[C]]
+    ; GFX9-UNSAFE: [[FMA7:%[0-9]+]]:_(s64) = G_FMA [[FMA6]], [[INT1]], [[INT1]]
+    ; GFX9-UNSAFE: [[FMA8:%[0-9]+]]:_(s64) = G_FMA [[FNEG1]], [[FMA7]], [[C]]
+    ; GFX9-UNSAFE: [[FMA9:%[0-9]+]]:_(s64) = G_FMA [[FMA8]], [[FMA7]], [[FMA7]]
+    ; GFX9-UNSAFE: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[UV1]], [[FMA9]]
+    ; GFX9-UNSAFE: [[FMA10:%[0-9]+]]:_(s64) = G_FMA [[FNEG1]], [[FMUL1]], [[UV1]]
+    ; GFX9-UNSAFE: [[FMA11:%[0-9]+]]:_(s64) = G_FMA [[FMA10]], [[FMA9]], [[FMUL1]]
+    ; GFX9-UNSAFE: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMA5]](s64), [[FMA11]](s64)
     ; GFX9-UNSAFE: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX10-LABEL: name: test_fdiv_v2s64
     ; GFX10: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX10: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX10: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
     ; GFX10: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX10: [[INT:%[0-9]+]]:_(s64), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[UV]](s64), [[UV2]](s64), 0
bb.0:
     ; GFX9: [[INT3:%[0-9]+]]:_(s64), [[INT4:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[C]](s64), [[COPY]](s64), 1
     ; GFX9: [[FMA3:%[0-9]+]]:_(s64) = G_FMA [[FMA1]], [[FMA2]], [[FMA1]]
     ; GFX9: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INT3]], [[FMA3]]
     ; GFX9: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[INT3]]
     ; GFX9: [[INT5:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fmas), [[FMA4]](s64), [[FMA3]](s64), [[FMUL]](s64), [[INT4]](s1)
     ; GFX9: [[INT6:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[INT5]](s64), [[COPY]](s64), [[C]](s64)
     ; GFX9: $vgpr0_vgpr1 = COPY [[INT6]](s64)
     ; GFX9-UNSAFE-LABEL: name: test_fdiv_s64_constant_one_rcp
+    ; GFX9-UNSAFE: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX9-UNSAFE: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9-UNSAFE: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY]]
     ; GFX9-UNSAFE: [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[COPY]](s64)
-    ; GFX9-UNSAFE: $vgpr0_vgpr1 = COPY [[INT]](s64)
+    ; GFX9-UNSAFE: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT]], [[C]]
+    ; GFX9-UNSAFE: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FMA]], [[INT]], [[INT]]
+    ; GFX9-UNSAFE: [[FMA2:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMA1]], [[C]]
+    ; GFX9-UNSAFE: [[FMA3:%[0-9]+]]:_(s64) = G_FMA [[FMA2]], [[FMA1]], [[FMA1]]
+    ; GFX9-UNSAFE: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[C]], [[FMA3]]
+    ; GFX9-UNSAFE: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[C]]
+    ; GFX9-UNSAFE: [[FMA5:%[0-9]+]]:_(s64) = G_FMA [[FMA4]], [[FMA3]], [[FMUL]]
+    ; GFX9-UNSAFE: $vgpr0_vgpr1 = COPY [[FMA5]](s64)
     ; GFX10-LABEL: name: test_fdiv_s64_constant_one_rcp
     ; GFX10: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX10: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX10: [[INT:%[0-9]+]]:_(s64), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[C]](s64), [[COPY]](s64), 0
     ; GFX10: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[INT]]
     ; GFX10: [[INT2:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[INT]](s64)
     ; GFX10: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT2]], [[C]]
     ; GFX10: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[INT2]], [[FMA]], [[INT2]]
bb.0:
     ; GFX9: [[INT3:%[0-9]+]]:_(s64), [[INT4:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[C]](s64), [[COPY]](s64), 1
     ; GFX9: [[FMA3:%[0-9]+]]:_(s64) = G_FMA [[FMA1]], [[FMA2]], [[FMA1]]
     ; GFX9: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INT3]], [[FMA3]]
     ; GFX9: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[INT3]]
     ; GFX9: [[INT5:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fmas), [[FMA4]](s64), [[FMA3]](s64), [[FMUL]](s64), [[INT4]](s1)
     ; GFX9: [[INT6:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[INT5]](s64), [[COPY]](s64), [[C]](s64)
     ; GFX9: $vgpr0_vgpr1 = COPY [[INT6]](s64)
     ; GFX9-UNSAFE-LABEL: name: test_fdiv_s64_constant_negative_one_rcp
+    ; GFX9-UNSAFE: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double -1.000000e+00
     ; GFX9-UNSAFE: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9-UNSAFE: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY]]
-    ; GFX9-UNSAFE: [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[FNEG]](s64)
-    ; GFX9-UNSAFE: $vgpr0_vgpr1 = COPY [[INT]](s64)
+    ; GFX9-UNSAFE: [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; GFX9-UNSAFE: [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[COPY]](s64)
+    ; GFX9-UNSAFE: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT]], [[C1]]
+    ; GFX9-UNSAFE: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FMA]], [[INT]], [[INT]]
+    ; GFX9-UNSAFE: [[FMA2:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMA1]], [[C1]]
+    ; GFX9-UNSAFE: [[FMA3:%[0-9]+]]:_(s64) = G_FMA [[FMA2]], [[FMA1]], [[FMA1]]
+    ; GFX9-UNSAFE: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[C]], [[FMA3]]
+    ; GFX9-UNSAFE: [[FMA4:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[FMUL]], [[C]]
+    ; GFX9-UNSAFE: [[FMA5:%[0-9]+]]:_(s64) = G_FMA [[FMA4]], [[FMA3]], [[FMUL]]
+    ; GFX9-UNSAFE: $vgpr0_vgpr1 = COPY [[FMA5]](s64)
     ; GFX10-LABEL: name: test_fdiv_s64_constant_negative_one_rcp
     ; GFX10: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double -1.000000e+00
     ; GFX10: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX10: [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; GFX10: [[INT:%[0-9]+]]:_(s64), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[C]](s64), [[COPY]](s64), 0
     ; GFX10: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[INT]]
     ; GFX10: [[INT2:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[INT]](s64)
     ; GFX10: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FNEG]], [[INT2]], [[C1]]

llvm/test/CodeGen/AMDGPU/fdiv.f64.ll

define amdgpu_kernel void @fdiv_f64(double addrspace(1)* %out, double addrspace(1)* %in) #0 {
   %gep.1 = getelementptr double, double addrspace(1)* %in, i32 1
   %num = load volatile double, double addrspace(1)* %in
   %den = load volatile double, double addrspace(1)* %gep.1
   %result = fdiv double %num, %den
   store double %result, double addrspace(1)* %out
   ret void
 }
 
+; GCN-LABEL: {{^}}v_fdiv_f64_afn:
+; GCN: v_rcp_f64_e32 v[4:5], v[2:3]
+; GCN: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GCN: v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GCN: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GCN: v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GCN: v_mul_f64 v[6:7], v[0:1], v[4:5]
+; GCN: v_fma_f64 v[0:1], -v[2:3], v[6:7], v[0:1]
+; GCN: v_fma_f64 v[0:1], v[0:1], v[4:5], v[6:7]
+; GCN: s_setpc_b64
+define double @v_fdiv_f64_afn(double %x, double %y) #0 {
+  %result = fdiv afn double %x, %y
+  ret double %result
+}
+
+; GCN-LABEL: {{^}}v_rcp_f64_afn:
+; GCN: v_rcp_f64_e32 v[2:3], v[0:1]
+; GCN: v_fma_f64 v[4:5], -v[0:1], v[2:3], 1.0
+; GCN: v_fma_f64 v[2:3], v[4:5], v[2:3], v[2:3]
+; GCN: v_fma_f64 v[4:5], -v[0:1], v[2:3], 1.0
+; GCN: v_fma_f64 v[2:3], v[4:5], v[2:3], v[2:3]
+; GCN: v_fma_f64 v[0:1], -v[0:1], v[2:3], 1.0
+; GCN: v_fma_f64 v[0:1], v[0:1], v[2:3], v[2:3]
+; GCN: s_setpc_b64
+define double @v_rcp_f64_afn(double %x) #0 {
+  %result = fdiv afn double 1.0, %x
+  ret double %result
+}
+
 ; GCN-LABEL: {{^}}fdiv_f64_s_v:
 define amdgpu_kernel void @fdiv_f64_s_v(double addrspace(1)* %out, double addrspace(1)* %in, double %num) #0 {
   %den = load double, double addrspace(1)* %in
   %result = fdiv double %num, %den
   store double %result, double addrspace(1)* %out
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/frem.ll

; VI-NEXT:    s_endpgm
    %r2 = frem double %r0, %r1
    store double %r2, double addrspace(1)* %out, align 8
    ret void
 }
 
 define amdgpu_kernel void @fast_frem_f64(double addrspace(1)* %out, double addrspace(1)* %in1,
 ; SI-LABEL: fast_frem_f64:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s15, 0xf000
+; SI-NEXT:    s_mov_b32 s14, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s8
-; SI-NEXT:    s_mov_b32 s5, s9
-; SI-NEXT:    s_mov_b32 s0, s10
-; SI-NEXT:    s_mov_b32 s1, s11
-; SI-NEXT:    s_mov_b32 s2, s6
-; SI-NEXT:    s_mov_b32 s3, s7
-; SI-NEXT:    s_mov_b32 s14, s6
-; SI-NEXT:    s_mov_b32 s15, s7
+; SI-NEXT:    s_mov_b32 s12, s4
+; SI-NEXT:    s_mov_b32 s13, s5
+; SI-NEXT:    s_mov_b32 s0, s6
+; SI-NEXT:    s_mov_b32 s1, s7
+; SI-NEXT:    s_mov_b32 s2, s14
+; SI-NEXT:    s_mov_b32 s3, s15
+; SI-NEXT:    s_mov_b32 s10, s14
+; SI-NEXT:    s_mov_b32 s11, s15
 ; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[0:3], 0
-; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[8:11], 0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], v[0:1]
-; SI-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
-; SI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
-; SI-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
-; SI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
-; SI-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
-; SI-NEXT:    v_div_scale_f64 v[8:9], s[0:1], v[0:1], v[2:3], v[0:1]
-; SI-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
-; SI-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[10:11], v[8:9]
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; SI-NEXT:    v_cmp_eq_u32_e64 s[0:1], v1, v9
-; SI-NEXT:    s_xor_b64 vcc, s[0:1], vcc
-; SI-NEXT:    s_nop 1
-; SI-NEXT:    v_div_fmas_f64 v[4:5], v[12:13], v[6:7], v[10:11]
-; SI-NEXT:    v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
+; SI-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; SI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; SI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; SI-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
+; SI-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
+; SI-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
 ; SI-NEXT:    v_bfe_u32 v6, v5, 20, 11
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 0xfffffc01, v6
 ; SI-NEXT:    s_mov_b32 s1, 0xfffff
-; SI-NEXT:    s_mov_b32 s0, s6
+; SI-NEXT:    s_mov_b32 s0, s14
 ; SI-NEXT:    v_lshr_b64 v[6:7], s[0:1], v8
 ; SI-NEXT:    v_not_b32_e32 v6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, v4, v6
 ; SI-NEXT:    v_not_b32_e32 v7, v7
 ; SI-NEXT:    v_and_b32_e32 v7, v5, v7
 ; SI-NEXT:    v_and_b32_e32 v9, 0x80000000, v5
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v8
 ; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], 51, v8
 ; SI-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[0:1]
 ; SI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[12:15], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: fast_frem_f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s11, 0xf000
 ; CI-NEXT:    s_mov_b32 s10, -1
 ; CI-NEXT:    s_mov_b32 s2, s10
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_mov_b32 s8, s4
 ; CI-NEXT:    s_mov_b32 s9, s5
 ; CI-NEXT:    s_mov_b32 s4, s6
 ; CI-NEXT:    s_mov_b32 s5, s7
 ; CI-NEXT:    s_mov_b32 s6, s10
 ; CI-NEXT:    s_mov_b32 s7, s11
 ; CI-NEXT:    s_mov_b32 s3, s11
 ; CI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; CI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], v[0:1]
-; CI-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
-; CI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
-; CI-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
-; CI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
-; CI-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
-; CI-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
-; CI-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
-; CI-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
-; CI-NEXT:    s_nop 1
-; CI-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
-; CI-NEXT:    v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
+; CI-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; CI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; CI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; CI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; CI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; CI-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
+; CI-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
+; CI-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
 ; CI-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
 ; CI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fast_frem_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[2:3]
-; VI-NEXT:    v_rcp_f64_e32 v[8:9], v[6:7]
-; VI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
-; VI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
-; VI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
-; VI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
-; VI-NEXT:    v_div_scale_f64 v[10:11], vcc, v[2:3], v[4:5], v[2:3]
-; VI-NEXT:    v_mul_f64 v[12:13], v[10:11], v[8:9]
-; VI-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
-; VI-NEXT:    s_nop 1
-; VI-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
-; VI-NEXT:    v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[2:3]
+; VI-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; VI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; VI-NEXT:    v_fma_f64 v[6:7], v[8:9], v[6:7], v[6:7]
+; VI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; VI-NEXT:    v_fma_f64 v[6:7], v[8:9], v[6:7], v[6:7]
+; VI-NEXT:    v_mul_f64 v[8:9], v[2:3], v[6:7]
+; VI-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[8:9], v[2:3]
+; VI-NEXT:    v_fma_f64 v[6:7], v[10:11], v[6:7], v[8:9]
 ; VI-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
 ; VI-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3]
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
                       double addrspace(1)* %in2) #0 {
    %r0 = load double, double addrspace(1)* %in1, align 8
    %r1 = load double, double addrspace(1)* %in2, align 8
    %r2 = frem fast double %r0, %r1

 ; SI-NEXT:    s_mov_b32 s2, s14
 ; SI-NEXT:    s_mov_b32 s3, s15
 ; SI-NEXT:    s_mov_b32 s10, s14
 ; SI-NEXT:    s_mov_b32 s11, s15
 ; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[8:11], 0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
-; SI-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; SI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; SI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; SI-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
+; SI-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
+; SI-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
 ; SI-NEXT:    v_bfe_u32 v6, v5, 20, 11
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 0xfffffc01, v6
 ; SI-NEXT:    s_mov_b32 s1, 0xfffff
 ; SI-NEXT:    s_mov_b32 s0, s14
 ; SI-NEXT:    v_lshr_b64 v[6:7], s[0:1], v8
 ; SI-NEXT:    v_not_b32_e32 v6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, v4, v6
 ; SI-NEXT:    v_not_b32_e32 v7, v7

 ; CI-NEXT:    s_mov_b32 s5, s7
 ; CI-NEXT:    s_mov_b32 s6, s10
 ; CI-NEXT:    s_mov_b32 s7, s11
 ; CI-NEXT:    s_mov_b32 s3, s11
 ; CI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; CI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
-; CI-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; CI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; CI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; CI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; CI-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; CI-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
+; CI-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
+; CI-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
 ; CI-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
 ; CI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: unsafe_frem_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
-; VI-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; VI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; VI-NEXT:    v_fma_f64 v[6:7], v[8:9], v[6:7], v[6:7]
+; VI-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; VI-NEXT:    v_fma_f64 v[6:7], v[8:9], v[6:7], v[6:7]
+; VI-NEXT:    v_mul_f64 v[8:9], v[2:3], v[6:7]
+; VI-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[8:9], v[2:3]
+; VI-NEXT:    v_fma_f64 v[6:7], v[10:11], v[6:7], v[8:9]
 ; VI-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
 ; VI-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3]
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
                              double addrspace(1)* %in2) #1 {
    %r0 = load double, double addrspace(1)* %in1, align 8
    %r1 = load double, double addrspace(1)* %in2, align 8
    %r2 = frem afn double %r0, %r1

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll

 ; SI: v_div_scale_f64
 define amdgpu_kernel void @rcp_pat_f64(double addrspace(1)* %out, double %src) #1 {
   %rcp = fdiv double 1.0, %src
   store double %rcp, double addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}unsafe_rcp_pat_f64:
-; SI: v_rcp_f64_e32 [[RESULT:v\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}
-; SI-NOT: [[RESULT]]
-; SI: buffer_store_dwordx2 [[RESULT]]
+; SI: v_rcp_f64
+; SI: v_fma_f64
+; SI: v_fma_f64
+; SI: v_fma_f64
+; SI: v_fma_f64
+; SI: v_fma_f64
+; SI: v_fma_f64
 define amdgpu_kernel void @unsafe_rcp_pat_f64(double addrspace(1)* %out, double %src) #2 {
   %rcp = fdiv double 1.0, %src
   store double %rcp, double addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}safe_rsq_rcp_pat_f64:
 ; SI-NOT: v_rsq_f64_e32

llvm/test/CodeGen/AMDGPU/rsq.ll

define amdgpu_kernel void @neg_rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   store float %div, float addrspace(1)* %out, align 4
   ret void
 }
 
 ; SI-LABEL: {{^}}neg_rsq_f64:
 ; SI-SAFE: v_sqrt_f64_e32
 ; SI-SAFE: v_div_scale_f64
 
-; SI-UNSAFE: v_sqrt_f64_e32 [[SQRT:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}
-; SI-UNSAFE: v_rcp_f64_e64 [[RCP:v\[[0-9]+:[0-9]+\]]], -[[SQRT]]
-; SI-UNSAFE: buffer_store_dwordx2 [[RCP]]
+; SI-UNSAFE: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]]
+; SI-UNSAFE: v_sqrt_f64_e32 [[SQRT:v\[[0-9]+:[0-9]+\]]], [[VAL]]
+; SI-UNSAFE: v_rsq_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], [[VAL]]
+; SI-UNSAFE: v_fma_f64 {{v\[[0-9]+:[0-9]+\]}}, -{{v\[[0-9]+:[0-9]+\]}}, [[RSQ]], 1.0
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
 define amdgpu_kernel void @neg_rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 {
   %val = load double, double addrspace(1)* %in, align 4
   %sqrt = call double @llvm.sqrt.f64(double %val)
   %div = fdiv double -1.0, %sqrt
   store double %div, double addrspace(1)* %out, align 4
   ret void
 }
 
define amdgpu_kernel void @neg_rsq_neg_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   store float %div, float addrspace(1)* %out, align 4
   ret void
 }
 
 ; SI-LABEL: {{^}}neg_rsq_neg_f64:
 ; SI-SAFE: v_sqrt_f64_e64 v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}}
 ; SI-SAFE: v_div_scale_f64
 
-; SI-UNSAFE: v_sqrt_f64_e64 [[SQRT:v\[[0-9]+:[0-9]+\]]], -v{{\[[0-9]+:[0-9]+\]}}
-; SI-UNSAFE: v_rcp_f64_e64 [[RCP:v\[[0-9]+:[0-9]+\]]], -[[SQRT]]
-; SI-UNSAFE: buffer_store_dwordx2 [[RCP]]
+; SI-UNSAFE: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]]
+; SI-UNSAFE-DAG: v_sqrt_f64_e64 [[SQRT:v\[[0-9]+:[0-9]+\]]], -[[VAL]]
+; SI-UNSAFE-DAG: v_xor_b32_e32 v[[HI:[0-9]+]], 0x80000000, v{{[0-9]+}}
+; SI-UNSAFE: v_rsq_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+}}:[[HI]]{{\]}}
+; SI-UNSAFE: v_fma_f64 {{v\[[0-9]+:[0-9]+\]}}, -{{v\[[0-9]+:[0-9]+\]}}, [[RSQ]], 1.0
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
+; SI-UNSAFE: v_fma_f64
 define amdgpu_kernel void @neg_rsq_neg_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 {
   %val = load double, double addrspace(1)* %in, align 4
   %val.fneg = fsub double -0.0, %val
   %sqrt = call double @llvm.sqrt.f64(double %val.fneg)
   %div = fdiv double -1.0, %sqrt
   store double %div, double addrspace(1)* %out, align 4
   ret void
 }
 
 !0 = !{float 2.500000e+00}
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }