[AMDGPU] Use fma for lowering frem

[foad]

Summary

This gives shorter f64 code and perhaps better accuracy.

[arsenm]

If it doesn’t actually help accuracy, maybe it should check for fast FMA

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

2091–2092

Could rename floor to trunc

[foad]

Rename misleading Floor -> Trunc.

[foad]

Rebase.

[foad]

If it doesn’t actually help accuracy, maybe it should check for fast FMA

I don't have any evidence that it improves accuracy but I'm not sure what the conditions should be for using FMA (I looked at SITargetLowering::isFMAFasterThanFMulAndFAdd but it is complicated) so I left it as a TODO.

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
 
   SDValue Res[2] = {
     Div,
     Rem
   };
   return DAG.getMergeValues(Res, DL);
 }
 
-// (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y))
+// (frem x, y) -> (fma (fneg (ftrunc (fdiv x, y))), y, x)
 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   EVT VT = Op.getValueType();
   SDValue X = Op.getOperand(0);
   SDValue Y = Op.getOperand(1);
 
   // TODO: Should this propagate fast-math-flags?
 
   SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
-  SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
-  SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
-
-  return DAG.getNode(ISD::FSUB, SL, VT, X, Mul);
+  SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
+  SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc);
+  // TODO: For f32 use FMAD instead if !hasFastFMA32?
+  return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X);
 }
 
 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   // result = trunc(src)
   // if (src > 0.0 && src != result)

llvm/test/CodeGen/AMDGPU/frem.ll

 ; SI-NEXT:    v_mul_f32_e32 v5, v2, v4
 ; SI-NEXT:    v_fma_f32 v6, -v3, v5, v2
 ; SI-NEXT:    v_fma_f32 v5, v6, v4, v5
 ; SI-NEXT:    v_fma_f32 v2, -v3, v5, v2
 ; SI-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; SI-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
 ; SI-NEXT:    v_div_fixup_f32 v2, v2, v1, v0
 ; SI-NEXT:    v_trunc_f32_e32 v2, v2
-; SI-NEXT:    v_mad_f32 v0, -v2, v1, v0
+; SI-NEXT:    v_fma_f32 v0, -v2, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: frem_f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s11, 0xf000

 ; CI-NEXT:    v_mul_f32_e32 v5, v2, v4
 ; CI-NEXT:    v_fma_f32 v6, -v3, v5, v2
 ; CI-NEXT:    v_fma_f32 v5, v6, v4, v5
 ; CI-NEXT:    v_fma_f32 v2, -v3, v5, v2
 ; CI-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; CI-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
 ; CI-NEXT:    v_div_fixup_f32 v2, v2, v1, v0
 ; CI-NEXT:    v_trunc_f32_e32 v2, v2
-; CI-NEXT:    v_mad_f32 v0, -v2, v1, v0
+; CI-NEXT:    v_fma_f32 v0, -v2, v1, v0
 ; CI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: frem_f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)

 ; VI-NEXT:    v_mul_f32_e32 v7, v3, v6
 ; VI-NEXT:    v_fma_f32 v8, -v5, v7, v3
 ; VI-NEXT:    v_fma_f32 v7, v8, v6, v7
 ; VI-NEXT:    v_fma_f32 v3, -v5, v7, v3
 ; VI-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; VI-NEXT:    v_div_fmas_f32 v3, v3, v6, v7
 ; VI-NEXT:    v_div_fixup_f32 v3, v3, v2, v4
 ; VI-NEXT:    v_trunc_f32_e32 v3, v3
-; VI-NEXT:    v_mad_f32 v2, -v3, v2, v4
+; VI-NEXT:    v_fma_f32 v2, -v3, v2, v4
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
                       float addrspace(1)* %in2) #0 {
    %gep2 = getelementptr float, float addrspace(1)* %in2, i32 4
    %r0 = load float, float addrspace(1)* %in1, align 4
    %r1 = load float, float addrspace(1)* %gep2, align 4
    %r2 = frem float %r0, %r1
    store float %r2, float addrspace(1)* %out, align 4

 ; SI-NEXT:    s_mov_b32 s10, s2
 ; SI-NEXT:    s_mov_b32 s11, s3
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    buffer_load_dword v1, off, s[8:11], 0 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_rcp_f32_e32 v2, v1
 ; SI-NEXT:    v_mul_f32_e32 v2, v0, v2
 ; SI-NEXT:    v_trunc_f32_e32 v2, v2
-; SI-NEXT:    v_mad_f32 v0, -v2, v1, v0
+; SI-NEXT:    v_fma_f32 v0, -v2, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: unsafe_frem_f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s11, 0xf000
 ; CI-NEXT:    s_mov_b32 s10, -1
 ; CI-NEXT:    s_mov_b32 s2, s10
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_mov_b32 s8, s4
 ; CI-NEXT:    s_mov_b32 s9, s5
 ; CI-NEXT:    s_mov_b32 s4, s6
 ; CI-NEXT:    s_mov_b32 s5, s7
 ; CI-NEXT:    s_mov_b32 s6, s10
 ; CI-NEXT:    s_mov_b32 s7, s11
 ; CI-NEXT:    s_mov_b32 s3, s11
 ; CI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; CI-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:16
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_rcp_f32_e32 v2, v1
 ; CI-NEXT:    v_mul_f32_e32 v2, v0, v2
 ; CI-NEXT:    v_trunc_f32_e32 v2, v2
-; CI-NEXT:    v_mad_f32 v0, -v2, v1, v0
+; CI-NEXT:    v_fma_f32 v0, -v2, v1, v0
 ; CI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: unsafe_frem_f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    s_add_u32 s0, s0, 16
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    flat_load_dword v4, v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_load_dword v2, v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_rcp_f32_e32 v3, v2
 ; VI-NEXT:    v_mul_f32_e32 v3, v4, v3
 ; VI-NEXT:    v_trunc_f32_e32 v3, v3
-; VI-NEXT:    v_mad_f32 v2, -v3, v2, v4
+; VI-NEXT:    v_fma_f32 v2, -v3, v2, v4
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
                              float addrspace(1)* %in2) #1 {
    %gep2 = getelementptr float, float addrspace(1)* %in2, i32 4
    %r0 = load float, float addrspace(1)* %in1, align 4
    %r1 = load float, float addrspace(1)* %gep2, align 4
    %r2 = frem float %r0, %r1
    store float %r2, float addrspace(1)* %out, align 4

 ; SI-NEXT:    v_and_b32_e32 v7, v5, v7
 ; SI-NEXT:    v_and_b32_e32 v9, 0x80000000, v5
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v8
 ; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], 51, v8
 ; SI-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[0:1]
-; SI-NEXT:    v_mul_f64 v[2:3], v[4:5], v[2:3]
-; SI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; SI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: frem_f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s11, 0xf000

 ; CI-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; CI-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
 ; CI-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
 ; CI-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
 ; CI-NEXT:    s_nop 1
 ; CI-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
 ; CI-NEXT:    v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
 ; CI-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
-; CI-NEXT:    v_mul_f64 v[2:3], v[4:5], v[2:3]
-; CI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; CI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: frem_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)

 ; VI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; VI-NEXT:    v_div_scale_f64 v[10:11], vcc, v[2:3], v[4:5], v[2:3]
 ; VI-NEXT:    v_mul_f64 v[12:13], v[10:11], v[8:9]
 ; VI-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
 ; VI-NEXT:    s_nop 1
 ; VI-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
 ; VI-NEXT:    v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[2:3]
 ; VI-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
-; VI-NEXT:    v_mul_f64 v[4:5], v[6:7], v[4:5]
-; VI-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; VI-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3]
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
                       double addrspace(1)* %in2) #0 {
    %r0 = load double, double addrspace(1)* %in1, align 8
    %r1 = load double, double addrspace(1)* %in2, align 8
    %r2 = frem double %r0, %r1
    store double %r2, double addrspace(1)* %out, align 8
    ret void

 ; SI-NEXT:    v_and_b32_e32 v7, v5, v7
 ; SI-NEXT:    v_and_b32_e32 v9, 0x80000000, v5
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v8
 ; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], 51, v8
 ; SI-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[0:1]
-; SI-NEXT:    v_mul_f64 v[2:3], v[4:5], v[2:3]
-; SI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; SI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[12:15], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: unsafe_frem_f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s11, 0xf000
 ; CI-NEXT:    s_mov_b32 s10, -1
 ; CI-NEXT:    s_mov_b32 s2, s10
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_mov_b32 s8, s4
 ; CI-NEXT:    s_mov_b32 s9, s5
 ; CI-NEXT:    s_mov_b32 s4, s6
 ; CI-NEXT:    s_mov_b32 s5, s7
 ; CI-NEXT:    s_mov_b32 s6, s10
 ; CI-NEXT:    s_mov_b32 s7, s11
 ; CI-NEXT:    s_mov_b32 s3, s11
 ; CI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
 ; CI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
 ; CI-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
 ; CI-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
-; CI-NEXT:    v_mul_f64 v[2:3], v[4:5], v[2:3]
-; CI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; CI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: unsafe_frem_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
 ; VI-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
 ; VI-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
-; VI-NEXT:    v_mul_f64 v[4:5], v[6:7], v[4:5]
-; VI-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; VI-NEXT:    v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3]
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
                              double addrspace(1)* %in2) #1 {
    %r0 = load double, double addrspace(1)* %in1, align 8
    %r1 = load double, double addrspace(1)* %in2, align 8
    %r2 = frem double %r0, %r1
    store double %r2, double addrspace(1)* %out, align 8
    ret void

 ; SI-NEXT:    v_mul_f32_e32 v7, v4, v6
 ; SI-NEXT:    v_fma_f32 v8, -v5, v7, v4
 ; SI-NEXT:    v_fma_f32 v7, v8, v6, v7
 ; SI-NEXT:    v_fma_f32 v4, -v5, v7, v4
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; SI-NEXT:    v_div_fmas_f32 v4, v4, v6, v7
 ; SI-NEXT:    v_div_fixup_f32 v4, v4, v3, v1
 ; SI-NEXT:    v_trunc_f32_e32 v4, v4
-; SI-NEXT:    v_mad_f32 v1, -v4, v3, v1
+; SI-NEXT:    v_fma_f32 v1, -v4, v3, v1
 ; SI-NEXT:    v_div_scale_f32 v3, vcc, v0, v2, v0
 ; SI-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
 ; SI-NEXT:    v_rcp_f32_e32 v5, v4
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; SI-NEXT:    v_fma_f32 v6, -v4, v5, 1.0
 ; SI-NEXT:    v_fma_f32 v5, v6, v5, v5
 ; SI-NEXT:    v_mul_f32_e32 v6, v3, v5
 ; SI-NEXT:    v_fma_f32 v7, -v4, v6, v3
 ; SI-NEXT:    v_fma_f32 v6, v7, v5, v6
 ; SI-NEXT:    v_fma_f32 v3, -v4, v6, v3
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; SI-NEXT:    v_div_fmas_f32 v3, v3, v5, v6
 ; SI-NEXT:    v_div_fixup_f32 v3, v3, v2, v0
 ; SI-NEXT:    v_trunc_f32_e32 v3, v3
-; SI-NEXT:    v_mad_f32 v0, -v3, v2, v0
+; SI-NEXT:    v_fma_f32 v0, -v3, v2, v0
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: frem_v2f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s3, 0xf000

 ; CI-NEXT:    v_mul_f32_e32 v7, v4, v6
 ; CI-NEXT:    v_fma_f32 v8, -v5, v7, v4
 ; CI-NEXT:    v_fma_f32 v7, v8, v6, v7
 ; CI-NEXT:    v_fma_f32 v4, -v5, v7, v4
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v4, v4, v6, v7
 ; CI-NEXT:    v_div_fixup_f32 v4, v4, v3, v1
 ; CI-NEXT:    v_trunc_f32_e32 v4, v4
-; CI-NEXT:    v_mad_f32 v1, -v4, v3, v1
+; CI-NEXT:    v_fma_f32 v1, -v4, v3, v1
 ; CI-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
 ; CI-NEXT:    v_div_scale_f32 v3, vcc, v0, v2, v0
 ; CI-NEXT:    v_rcp_f32_e32 v5, v4
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v6, -v4, v5, 1.0
 ; CI-NEXT:    v_fma_f32 v5, v6, v5, v5
 ; CI-NEXT:    v_mul_f32_e32 v6, v3, v5
 ; CI-NEXT:    v_fma_f32 v7, -v4, v6, v3
 ; CI-NEXT:    v_fma_f32 v6, v7, v5, v6
 ; CI-NEXT:    v_fma_f32 v3, -v4, v6, v3
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v3, v3, v5, v6
 ; CI-NEXT:    v_div_fixup_f32 v3, v3, v2, v0
 ; CI-NEXT:    v_trunc_f32_e32 v3, v3
-; CI-NEXT:    v_mad_f32 v0, -v3, v2, v0
+; CI-NEXT:    v_fma_f32 v0, -v3, v2, v0
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: frem_v2f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_mov_b32 s2, 3

 ; VI-NEXT:    v_mul_f32_e32 v9, v6, v8
 ; VI-NEXT:    v_fma_f32 v10, -v7, v9, v6
 ; VI-NEXT:    v_fma_f32 v9, v10, v8, v9
 ; VI-NEXT:    v_fma_f32 v6, -v7, v9, v6
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s3
 ; VI-NEXT:    v_div_fmas_f32 v6, v6, v8, v9
 ; VI-NEXT:    v_div_fixup_f32 v6, v6, v5, v3
 ; VI-NEXT:    v_trunc_f32_e32 v6, v6
-; VI-NEXT:    v_mad_f32 v3, -v6, v5, v3
+; VI-NEXT:    v_fma_f32 v3, -v6, v5, v3
 ; VI-NEXT:    v_div_scale_f32 v6, s[0:1], v4, v4, v2
 ; VI-NEXT:    v_div_scale_f32 v5, vcc, v2, v4, v2
 ; VI-NEXT:    v_rcp_f32_e32 v7, v6
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s2
 ; VI-NEXT:    v_fma_f32 v8, -v6, v7, 1.0
 ; VI-NEXT:    v_fma_f32 v7, v8, v7, v7
 ; VI-NEXT:    v_mul_f32_e32 v8, v5, v7
 ; VI-NEXT:    v_fma_f32 v9, -v6, v8, v5
 ; VI-NEXT:    v_fma_f32 v8, v9, v7, v8
 ; VI-NEXT:    v_fma_f32 v5, -v6, v8, v5
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s3
 ; VI-NEXT:    v_div_fmas_f32 v5, v5, v7, v8
 ; VI-NEXT:    v_div_fixup_f32 v5, v5, v4, v2
 ; VI-NEXT:    v_trunc_f32_e32 v5, v5
-; VI-NEXT:    v_mad_f32 v2, -v5, v4, v2
+; VI-NEXT:    v_fma_f32 v2, -v5, v4, v2
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
                         <2 x float> addrspace(1)* %in2) #0 {
    %gep2 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in2, i32 4
    %r0 = load <2 x float>, <2 x float> addrspace(1)* %in1, align 8
    %r1 = load <2 x float>, <2 x float> addrspace(1)* %gep2, align 8
    %r2 = frem <2 x float> %r0, %r1
    store <2 x float> %r2, <2 x float> addrspace(1)* %out, align 8

 ; SI-NEXT:    v_mul_f32_e32 v11, v8, v10
 ; SI-NEXT:    v_fma_f32 v12, -v9, v11, v8
 ; SI-NEXT:    v_fma_f32 v11, v12, v10, v11
 ; SI-NEXT:    v_fma_f32 v8, -v9, v11, v8
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; SI-NEXT:    v_div_fmas_f32 v8, v8, v10, v11
 ; SI-NEXT:    v_div_fixup_f32 v8, v8, v7, v3
 ; SI-NEXT:    v_trunc_f32_e32 v8, v8
-; SI-NEXT:    v_mad_f32 v3, -v8, v7, v3
+; SI-NEXT:    v_fma_f32 v3, -v8, v7, v3
 ; SI-NEXT:    v_div_scale_f32 v7, vcc, v2, v6, v2
 ; SI-NEXT:    v_div_scale_f32 v8, s[4:5], v6, v6, v2
 ; SI-NEXT:    v_rcp_f32_e32 v9, v8
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; SI-NEXT:    v_fma_f32 v10, -v8, v9, 1.0
 ; SI-NEXT:    v_fma_f32 v9, v10, v9, v9
 ; SI-NEXT:    v_mul_f32_e32 v10, v7, v9
 ; SI-NEXT:    v_fma_f32 v11, -v8, v10, v7
 ; SI-NEXT:    v_fma_f32 v10, v11, v9, v10
 ; SI-NEXT:    v_fma_f32 v7, -v8, v10, v7
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; SI-NEXT:    v_div_fmas_f32 v7, v7, v9, v10
 ; SI-NEXT:    v_div_fixup_f32 v7, v7, v6, v2
 ; SI-NEXT:    v_trunc_f32_e32 v7, v7
-; SI-NEXT:    v_mad_f32 v2, -v7, v6, v2
+; SI-NEXT:    v_fma_f32 v2, -v7, v6, v2
 ; SI-NEXT:    v_div_scale_f32 v6, vcc, v1, v5, v1
 ; SI-NEXT:    v_div_scale_f32 v7, s[4:5], v5, v5, v1
 ; SI-NEXT:    v_rcp_f32_e32 v8, v7
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; SI-NEXT:    v_fma_f32 v9, -v7, v8, 1.0
 ; SI-NEXT:    v_fma_f32 v8, v9, v8, v8
 ; SI-NEXT:    v_mul_f32_e32 v9, v6, v8
 ; SI-NEXT:    v_fma_f32 v10, -v7, v9, v6
 ; SI-NEXT:    v_fma_f32 v9, v10, v8, v9
 ; SI-NEXT:    v_fma_f32 v6, -v7, v9, v6
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; SI-NEXT:    v_div_fmas_f32 v6, v6, v8, v9
 ; SI-NEXT:    v_div_fixup_f32 v6, v6, v5, v1
 ; SI-NEXT:    v_trunc_f32_e32 v6, v6
-; SI-NEXT:    v_mad_f32 v1, -v6, v5, v1
+; SI-NEXT:    v_fma_f32 v1, -v6, v5, v1
 ; SI-NEXT:    v_div_scale_f32 v5, vcc, v0, v4, v0
 ; SI-NEXT:    v_div_scale_f32 v6, s[4:5], v4, v4, v0
 ; SI-NEXT:    v_rcp_f32_e32 v7, v6
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; SI-NEXT:    v_fma_f32 v8, -v6, v7, 1.0
 ; SI-NEXT:    v_fma_f32 v7, v8, v7, v7
 ; SI-NEXT:    v_mul_f32_e32 v8, v5, v7
 ; SI-NEXT:    v_fma_f32 v9, -v6, v8, v5
 ; SI-NEXT:    v_fma_f32 v8, v9, v7, v8
 ; SI-NEXT:    v_fma_f32 v5, -v6, v8, v5
 ; SI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; SI-NEXT:    v_div_fmas_f32 v5, v5, v7, v8
 ; SI-NEXT:    v_div_fixup_f32 v5, v5, v4, v0
 ; SI-NEXT:    v_trunc_f32_e32 v5, v5
-; SI-NEXT:    v_mad_f32 v0, -v5, v4, v0
+; SI-NEXT:    v_fma_f32 v0, -v5, v4, v0
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: frem_v4f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s3, 0xf000

 ; CI-NEXT:    v_mul_f32_e32 v11, v8, v10
 ; CI-NEXT:    v_fma_f32 v12, -v9, v11, v8
 ; CI-NEXT:    v_fma_f32 v11, v12, v10, v11
 ; CI-NEXT:    v_fma_f32 v8, -v9, v11, v8
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v8, v8, v10, v11
 ; CI-NEXT:    v_div_fixup_f32 v8, v8, v7, v3
 ; CI-NEXT:    v_trunc_f32_e32 v8, v8
-; CI-NEXT:    v_mad_f32 v3, -v8, v7, v3
+; CI-NEXT:    v_fma_f32 v3, -v8, v7, v3
 ; CI-NEXT:    v_div_scale_f32 v8, s[4:5], v6, v6, v2
 ; CI-NEXT:    v_div_scale_f32 v7, vcc, v2, v6, v2
 ; CI-NEXT:    v_rcp_f32_e32 v9, v8
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v10, -v8, v9, 1.0
 ; CI-NEXT:    v_fma_f32 v9, v10, v9, v9
 ; CI-NEXT:    v_mul_f32_e32 v10, v7, v9
 ; CI-NEXT:    v_fma_f32 v11, -v8, v10, v7
 ; CI-NEXT:    v_fma_f32 v10, v11, v9, v10
 ; CI-NEXT:    v_fma_f32 v7, -v8, v10, v7
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v7, v7, v9, v10
 ; CI-NEXT:    v_div_fixup_f32 v7, v7, v6, v2
 ; CI-NEXT:    v_trunc_f32_e32 v7, v7
-; CI-NEXT:    v_mad_f32 v2, -v7, v6, v2
+; CI-NEXT:    v_fma_f32 v2, -v7, v6, v2
 ; CI-NEXT:    v_div_scale_f32 v7, s[4:5], v5, v5, v1
 ; CI-NEXT:    v_div_scale_f32 v6, vcc, v1, v5, v1
 ; CI-NEXT:    v_rcp_f32_e32 v8, v7
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v9, -v7, v8, 1.0
 ; CI-NEXT:    v_fma_f32 v8, v9, v8, v8
 ; CI-NEXT:    v_mul_f32_e32 v9, v6, v8
 ; CI-NEXT:    v_fma_f32 v10, -v7, v9, v6
 ; CI-NEXT:    v_fma_f32 v9, v10, v8, v9
 ; CI-NEXT:    v_fma_f32 v6, -v7, v9, v6
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v6, v6, v8, v9
 ; CI-NEXT:    v_div_fixup_f32 v6, v6, v5, v1
 ; CI-NEXT:    v_trunc_f32_e32 v6, v6
-; CI-NEXT:    v_mad_f32 v1, -v6, v5, v1
+; CI-NEXT:    v_fma_f32 v1, -v6, v5, v1
 ; CI-NEXT:    v_div_scale_f32 v6, s[4:5], v4, v4, v0
 ; CI-NEXT:    v_div_scale_f32 v5, vcc, v0, v4, v0
 ; CI-NEXT:    v_rcp_f32_e32 v7, v6
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v8, -v6, v7, 1.0
 ; CI-NEXT:    v_fma_f32 v7, v8, v7, v7
 ; CI-NEXT:    v_mul_f32_e32 v8, v5, v7
 ; CI-NEXT:    v_fma_f32 v9, -v6, v8, v5
 ; CI-NEXT:    v_fma_f32 v8, v9, v7, v8
 ; CI-NEXT:    v_fma_f32 v5, -v6, v8, v5
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v5, v5, v7, v8
 ; CI-NEXT:    v_div_fixup_f32 v5, v5, v4, v0
 ; CI-NEXT:    v_trunc_f32_e32 v5, v5
-; CI-NEXT:    v_mad_f32 v0, -v5, v4, v0
+; CI-NEXT:    v_fma_f32 v0, -v5, v4, v0
 ; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: frem_v4f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_mov_b32 s2, 3

 ; VI-NEXT:    v_mul_f32_e32 v13, v10, v12
 ; VI-NEXT:    v_fma_f32 v14, -v11, v13, v10
 ; VI-NEXT:    v_fma_f32 v13, v14, v12, v13
 ; VI-NEXT:    v_fma_f32 v10, -v11, v13, v10
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s3
 ; VI-NEXT:    v_div_fmas_f32 v10, v10, v12, v13
 ; VI-NEXT:    v_div_fixup_f32 v10, v10, v7, v3
 ; VI-NEXT:    v_trunc_f32_e32 v10, v10
-; VI-NEXT:    v_mad_f32 v3, -v10, v7, v3
+; VI-NEXT:    v_fma_f32 v3, -v10, v7, v3
 ; VI-NEXT:    v_div_scale_f32 v10, s[0:1], v6, v6, v2
 ; VI-NEXT:    v_div_scale_f32 v7, vcc, v2, v6, v2
 ; VI-NEXT:    v_rcp_f32_e32 v11, v10
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s2
 ; VI-NEXT:    v_fma_f32 v12, -v10, v11, 1.0
 ; VI-NEXT:    v_fma_f32 v11, v12, v11, v11
 ; VI-NEXT:    v_mul_f32_e32 v12, v7, v11
 ; VI-NEXT:    v_fma_f32 v13, -v10, v12, v7
 ; VI-NEXT:    v_fma_f32 v12, v13, v11, v12
 ; VI-NEXT:    v_fma_f32 v7, -v10, v12, v7
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s3
 ; VI-NEXT:    v_div_fmas_f32 v7, v7, v11, v12
 ; VI-NEXT:    v_div_fixup_f32 v7, v7, v6, v2
 ; VI-NEXT:    v_trunc_f32_e32 v7, v7
-; VI-NEXT:    v_mad_f32 v2, -v7, v6, v2
+; VI-NEXT:    v_fma_f32 v2, -v7, v6, v2
 ; VI-NEXT:    v_div_scale_f32 v7, s[0:1], v5, v5, v1
 ; VI-NEXT:    v_div_scale_f32 v6, vcc, v1, v5, v1
 ; VI-NEXT:    v_rcp_f32_e32 v10, v7
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s2
 ; VI-NEXT:    v_fma_f32 v11, -v7, v10, 1.0
 ; VI-NEXT:    v_fma_f32 v10, v11, v10, v10
 ; VI-NEXT:    v_mul_f32_e32 v11, v6, v10
 ; VI-NEXT:    v_fma_f32 v12, -v7, v11, v6
 ; VI-NEXT:    v_fma_f32 v11, v12, v10, v11
 ; VI-NEXT:    v_fma_f32 v6, -v7, v11, v6
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s3
 ; VI-NEXT:    v_div_fmas_f32 v6, v6, v10, v11
 ; VI-NEXT:    v_div_fixup_f32 v6, v6, v5, v1
 ; VI-NEXT:    v_trunc_f32_e32 v6, v6
-; VI-NEXT:    v_mad_f32 v1, -v6, v5, v1
+; VI-NEXT:    v_fma_f32 v1, -v6, v5, v1
 ; VI-NEXT:    v_div_scale_f32 v6, s[0:1], v4, v4, v0
 ; VI-NEXT:    v_div_scale_f32 v5, vcc, v0, v4, v0
 ; VI-NEXT:    v_rcp_f32_e32 v7, v6
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s2
 ; VI-NEXT:    v_fma_f32 v10, -v6, v7, 1.0
 ; VI-NEXT:    v_fma_f32 v7, v10, v7, v7
 ; VI-NEXT:    v_mul_f32_e32 v10, v5, v7
 ; VI-NEXT:    v_fma_f32 v11, -v6, v10, v5
 ; VI-NEXT:    v_fma_f32 v10, v11, v7, v10
 ; VI-NEXT:    v_fma_f32 v5, -v6, v10, v5
 ; VI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s3
 ; VI-NEXT:    v_div_fmas_f32 v5, v5, v7, v10
 ; VI-NEXT:    v_div_fixup_f32 v5, v5, v4, v0
 ; VI-NEXT:    v_trunc_f32_e32 v5, v5
-; VI-NEXT:    v_mad_f32 v0, -v5, v4, v0
+; VI-NEXT:    v_fma_f32 v0, -v5, v4, v0
 ; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; VI-NEXT:    s_endpgm
                         <4 x float> addrspace(1)* %in2) #0 {
    %gep2 = getelementptr <4 x float>, <4 x float> addrspace(1)* %in2, i32 4
    %r0 = load <4 x float>, <4 x float> addrspace(1)* %in1, align 16
    %r1 = load <4 x float>, <4 x float> addrspace(1)* %gep2, align 16
    %r2 = frem <4 x float> %r0, %r1
    store <4 x float> %r2, <4 x float> addrspace(1)* %out, align 16

 ; SI-NEXT:    s_brev_b32 s9, 1
 ; SI-NEXT:    v_and_b32_e32 v13, s9, v9
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v12
 ; SI-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], 51, v12
 ; SI-NEXT:    v_cndmask_b32_e64 v9, v11, v9, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v10, v10, 0, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v8, v10, v8, s[0:1]
-; SI-NEXT:    v_mul_f64 v[6:7], v[8:9], v[6:7]
-; SI-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
 ; SI-NEXT:    v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
 ; SI-NEXT:    v_rcp_f64_e32 v[8:9], v[6:7]
 ; SI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
 ; SI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; SI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
 ; SI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; SI-NEXT:    v_div_scale_f64 v[10:11], s[0:1], v[0:1], v[4:5], v[0:1]
 ; SI-NEXT:    v_mul_f64 v[12:13], v[10:11], v[8:9]

 ; SI-NEXT:    v_and_b32_e32 v9, v7, v9
 ; SI-NEXT:    v_and_b32_e32 v11, s9, v7
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v10
 ; SI-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], 51, v10
 ; SI-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v8, v8, 0, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v6, v8, v6, s[0:1]
-; SI-NEXT:    v_mul_f64 v[4:5], v[6:7], v[4:5]
-; SI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; SI-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: frem_v2f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; CI-NEXT:    s_mov_b32 s3, 0xf000

 ; CI-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
 ; CI-NEXT:    v_div_scale_f64 v[12:13], vcc, v[2:3], v[6:7], v[2:3]
 ; CI-NEXT:    v_mul_f64 v[14:15], v[12:13], v[10:11]
 ; CI-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[14:15], v[12:13]
 ; CI-NEXT:    s_nop 1
 ; CI-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15]
 ; CI-NEXT:    v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
 ; CI-NEXT:    v_trunc_f64_e32 v[8:9], v[8:9]
-; CI-NEXT:    v_mul_f64 v[6:7], v[8:9], v[6:7]
-; CI-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; CI-NEXT:    v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
 ; CI-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[4:5], v[4:5], v[0:1]
 ; CI-NEXT:    v_rcp_f64_e32 v[8:9], v[6:7]
 ; CI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
 ; CI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; CI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
 ; CI-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; CI-NEXT:    v_div_scale_f64 v[10:11], vcc, v[0:1], v[4:5], v[0:1]
 ; CI-NEXT:    v_mul_f64 v[12:13], v[10:11], v[8:9]
 ; CI-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
 ; CI-NEXT:    s_nop 1
 ; CI-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
 ; CI-NEXT:    v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
 ; CI-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
-; CI-NEXT:    v_mul_f64 v[4:5], v[6:7], v[4:5]
-; CI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; CI-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
 ; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: frem_v2f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)

 ; VI-NEXT:    v_fma_f64 v[12:13], v[12:13], v[14:15], v[12:13]
 ; VI-NEXT:    v_div_scale_f64 v[14:15], vcc, v[2:3], v[6:7], v[2:3]
 ; VI-NEXT:    v_mul_f64 v[16:17], v[14:15], v[12:13]
 ; VI-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[16:17], v[14:15]
 ; VI-NEXT:    s_nop 1
 ; VI-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[12:13], v[16:17]
 ; VI-NEXT:    v_div_fixup_f64 v[10:11], v[10:11], v[6:7], v[2:3]
 ; VI-NEXT:    v_trunc_f64_e32 v[10:11], v[10:11]
-; VI-NEXT:    v_mul_f64 v[6:7], v[10:11], v[6:7]
-; VI-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; VI-NEXT:    v_fma_f64 v[2:3], -v[10:11], v[6:7], v[2:3]
 ; VI-NEXT:    v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
 ; VI-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
 ; VI-NEXT:    v_fma_f64 v[12:13], -v[6:7], v[10:11], 1.0
 ; VI-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
 ; VI-NEXT:    v_fma_f64 v[12:13], -v[6:7], v[10:11], 1.0
 ; VI-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
 ; VI-NEXT:    v_div_scale_f64 v[12:13], vcc, v[0:1], v[4:5], v[0:1]
 ; VI-NEXT:    v_mul_f64 v[14:15], v[12:13], v[10:11]
 ; VI-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[14:15], v[12:13]
 ; VI-NEXT:    s_nop 1
 ; VI-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[14:15]
 ; VI-NEXT:    v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
 ; VI-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
-; VI-NEXT:    v_mul_f64 v[4:5], v[6:7], v[4:5]
-; VI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; VI-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
 ; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; VI-NEXT:    s_endpgm
                         <2 x double> addrspace(1)* %in2) #0 {
    %gep2 = getelementptr <2 x double>, <2 x double> addrspace(1)* %in2, i32 4
    %r0 = load <2 x double>, <2 x double> addrspace(1)* %in1, align 16
    %r1 = load <2 x double>, <2 x double> addrspace(1)* %gep2, align 16
    %r2 = frem <2 x double> %r0, %r1
    store <2 x double> %r2, <2 x double> addrspace(1)* %out, align 16
    ret void
 }
 
 attributes #0 = { nounwind "unsafe-fp-math"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind "unsafe-fp-math"="true" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }