AMDGPU: Swap order of operands in fadd/fsub combine

[arsenm]

Summary

FMA is canonicalized to constant in the middle operand. Do
the same so fmad matches and avoid an extra combine step.

[• tstellarAMD]

LGTM.

[arsenm]

r290313

lib/Target/AMDGPU/SIISelLowering.cpp

SDValue SITargetLowering::performFAddCombine(SDNode *N,
 
   // fadd (fadd (a, a), b) -> mad 2.0, a, b
   if (LHS.getOpcode() == ISD::FADD) {
     SDValue A = LHS.getOperand(0);
     if (A == LHS.getOperand(1)) {
       unsigned FusedOp = getFusedOpcode(DAG, N, LHS.getNode());
       if (FusedOp != 0) {
         const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
-        return DAG.getNode(FusedOp, SL, VT, Two, A, RHS);
+        return DAG.getNode(FusedOp, SL, VT, A, Two, RHS);
       }
     }
   }
 
   // fadd (b, fadd (a, a)) -> mad 2.0, a, b
   if (RHS.getOpcode() == ISD::FADD) {
     SDValue A = RHS.getOperand(0);
     if (A == RHS.getOperand(1)) {
       unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode());
       if (FusedOp != 0) {
         const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
-        return DAG.getNode(FusedOp, SL, VT, Two, A, LHS);
+        return DAG.getNode(FusedOp, SL, VT, A, Two, LHS);
       }
     }
   }
 
   return SDValue();
 }
 
 SDValue SITargetLowering::performFSubCombine(SDNode *N,
if (LHS.getOpcode() == ISD::FADD) {
     // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
     SDValue A = LHS.getOperand(0);
     if (A == LHS.getOperand(1)) {
       unsigned FusedOp = getFusedOpcode(DAG, N, LHS.getNode());
       if (FusedOp != 0){
         const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
         SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
 
-        return DAG.getNode(FusedOp, SL, VT, Two, A, NegRHS);
+        return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS);
       }
     }
   }
 
   if (RHS.getOpcode() == ISD::FADD) {
     // (fsub c, (fadd a, a)) -> mad -2.0, a, c
 
     SDValue A = RHS.getOperand(0);
     if (A == RHS.getOperand(1)) {
       unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode());
       if (FusedOp != 0){
         const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
-        return DAG.getNode(FusedOp, SL, VT, NegTwo, A, LHS);
+        return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS);
       }
     }
   }
 
   return SDValue();
 }
 
 SDValue SITargetLowering::performSetCCCombine(SDNode *N,

test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll

define void @multiple_fadd_use_test_f32(float addrspace(1)* %out, float %x, float %y, float %z) #0 {
   %a19 = fmul fast float %a18, %a17
   %a20 = fsub fast float 1.0, %a19
   store float %a20, float addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}multiple_use_fadd_fmac_f32:
 ; GCN-DAG: v_add_f32_e64 [[MUL2:v[0-9]+]], [[X:s[0-9]+]], s{{[0-9]+}}
-; GCN-DAG: v_mac_f32_e64 [[MAD:v[0-9]+]], 2.0, [[X]]
+; GCN-DAG: v_mac_f32_e64 [[MAD:v[0-9]+]], [[X]], 2.0
 ; GCN-DAG: buffer_store_dword [[MUL2]]
 ; GCN-DAG: buffer_store_dword [[MAD]]
 ; GCN: s_endpgm
 define void @multiple_use_fadd_fmac_f32(float addrspace(1)* %out, float %x, float %y) #0 {
   %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
   %mul2 = fmul fast float %x, 2.0
   %mad = fadd fast float %mul2, %y
   store volatile float %mul2, float addrspace(1)* %out
   store volatile float %mad, float addrspace(1)* %out.gep.1
   ret void
 }
 
 ; GCN-LABEL: {{^}}multiple_use_fadd_fmad_f32:
 ; GCN-DAG: v_add_f32_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |s{{[0-9]+}}|
-; GCN-DAG: v_mad_f32 [[MAD:v[0-9]+]], 2.0, |[[X]]|, v{{[0-9]+}}
+; GCN-DAG: v_mad_f32 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
 ; GCN-DAG: buffer_store_dword [[MUL2]]
 ; GCN-DAG: buffer_store_dword [[MAD]]
 ; GCN: s_endpgm
 define void @multiple_use_fadd_fmad_f32(float addrspace(1)* %out, float %x, float %y) #0 {
   %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
   %x.abs = call float @llvm.fabs.f32(float %x)
   %mul2 = fmul fast float %x.abs, 2.0
   %mad = fadd fast float %mul2, %y
   store volatile float %mul2, float addrspace(1)* %out
   store volatile float %mad, float addrspace(1)* %out.gep.1
   ret void
 }
 
 ; GCN-LABEL: {{^}}multiple_use_fadd_multi_fmad_f32:
-; GCN: v_mad_f32 {{v[0-9]+}}, 2.0, |[[X:s[0-9]+]]|, v{{[0-9]+}}
-; GCN: v_mad_f32 {{v[0-9]+}}, 2.0, |[[X]]|, v{{[0-9]+}}
+; GCN: v_mad_f32 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0, v{{[0-9]+}}
+; GCN: v_mad_f32 {{v[0-9]+}}, |[[X]]|, 2.0, v{{[0-9]+}}
 define void @multiple_use_fadd_multi_fmad_f32(float addrspace(1)* %out, float %x, float %y, float %z) #0 {
   %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
   %x.abs = call float @llvm.fabs.f32(float %x)
   %mul2 = fmul fast float %x.abs, 2.0
   %mad0 = fadd fast float %mul2, %y
   %mad1 = fadd fast float %mul2, %z
   store volatile float %mad0, float addrspace(1)* %out
   store volatile float %mad1, float addrspace(1)* %out.gep.1
define void @multiple_fadd_use_test_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
   %a19 = fmul fast half %a18, %a17
   %a20 = fsub fast half 1.0, %a19
   store half %a20, half addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}multiple_use_fadd_fmac_f16:
 ; GCN-DAG: v_add_f16_e64 [[MUL2:v[0-9]+]], [[X:s[0-9]+]], s{{[0-9]+}}
-; GCN-DAG: v_mac_f16_e64 [[MAD:v[0-9]+]], 2.0, [[X]]
+; GCN-DAG: v_mac_f16_e64 [[MAD:v[0-9]+]], [[X]], 2.0
 ; GCN-DAG: buffer_store_short [[MUL2]]
 ; GCN-DAG: buffer_store_short [[MAD]]
 ; GCN: s_endpgm
 define void @multiple_use_fadd_fmac_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 {
   %x = bitcast i16 %x.arg to half
   %y = bitcast i16 %y.arg to half
   %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1
   %mul2 = fmul fast half %x, 2.0
   %mad = fadd fast half %mul2, %y
   store volatile half %mul2, half addrspace(1)* %out
   store volatile half %mad, half addrspace(1)* %out.gep.1
   ret void
 }
 
 ; GCN-LABEL: {{^}}multiple_use_fadd_fmad_f16:
 ; GCN-DAG: v_add_f16_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |s{{[0-9]+}}|
-; GCN-DAG: v_mad_f16 [[MAD:v[0-9]+]], 2.0, |[[X]]|, v{{[0-9]+}}
+; GCN-DAG: v_mad_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
 ; GCN-DAG: buffer_store_short [[MUL2]]
 ; GCN-DAG: buffer_store_short [[MAD]]
 ; GCN: s_endpgm
 define void @multiple_use_fadd_fmad_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 {
   %x = bitcast i16 %x.arg to half
   %y = bitcast i16 %y.arg to half
   %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1
   %x.abs = call half @llvm.fabs.f16(half %x)
   %mul2 = fmul fast half %x.abs, 2.0
   %mad = fadd fast half %mul2, %y
   store volatile half %mul2, half addrspace(1)* %out
   store volatile half %mad, half addrspace(1)* %out.gep.1
   ret void
 }
 
 ; GCN-LABEL: {{^}}multiple_use_fadd_multi_fmad_f16:
-; GCN: v_mad_f16 {{v[0-9]+}}, 2.0, |[[X:s[0-9]+]]|, v{{[0-9]+}}
-; GCN: v_mad_f16 {{v[0-9]+}}, 2.0, |[[X]]|, v{{[0-9]+}}
+; GCN: v_mad_f16 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0, v{{[0-9]+}}
+; GCN: v_mad_f16 {{v[0-9]+}}, |[[X]]|, 2.0, v{{[0-9]+}}
 define void @multiple_use_fadd_multi_fmad_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
   %x = bitcast i16 %x.arg to half
   %y = bitcast i16 %y.arg to half
   %z = bitcast i16 %z.arg to half
   %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1
   %x.abs = call half @llvm.fabs.f16(half %x)
   %mul2 = fmul fast half %x.abs, 2.0
   %mad0 = fadd fast half %mul2, %y

test/CodeGen/AMDGPU/fmuladd.f16.ll

define void @fmuladd_2.0_neg_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 {
   %r3 = tail call half @llvm.fmuladd.f16(half 2.0, half %r1.fneg, half %r2)
   store half %r3, half addrspace(1)* %gep.out
   ret void
 }
 
 ; GCN-LABEL: {{^}}fmuladd_2.0_a_neg_b_f16
 ; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]],
-; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], 2.0, [[R1]], -[[R2]]
+; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 ; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 ; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define void @fmuladd_2.0_a_neg_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid
   %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1
   %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid
 
define void @fsub_c_fadd_a_a_f16(half addrspace(1)* %out, half addrspace(1)* %in) {
   store half %r3, half addrspace(1)* %gep.out
   ret void
 }
 
 ; GCN-LABEL: {{^}}fsub_fadd_a_a_c_f16:
 ; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]],
 
-; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], 2.0, [[R1]], -[[R2]]
+; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 
 ; VI-DENORM-CONTRACT: v_fma_f16 [[R2]], [[R1]], 2.0, -[[R2]]
 
 ; VI-DENORM-STRICT: v_add_f16_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; VI-DENORM-STRICT: v_subrev_f16_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]]
 
 ; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define void @fsub_fadd_a_a_c_f16(half addrspace(1)* %out, half addrspace(1)* %in) {

test/CodeGen/AMDGPU/fmuladd.f32.ll

define void @fmuladd_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
   %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1.fneg, float %r2)
   store float %r3, float addrspace(1)* %gep.out
   ret void
 }
 
 ; GCN-LABEL: {{^}}fmuladd_2.0_a_neg_b_f32:
 ; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]],
-; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], -[[R2]]
+; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 ; SI-FLUSH: buffer_store_dword [[RESULT]]
 ; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 
 ; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 
 ; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; GCN-DENORM-SLOWFMA: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]]
 
define void @fsub_c_fadd_a_a_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
 
   store float %r3, float addrspace(1)* %gep.out
   ret void
 }
 
 ; GCN-LABEL: {{^}}fsub_fadd_a_a_c_f32:
 ; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]],
-; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], -[[R2]]
+; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 
 ; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 
 ; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; GCN-DENORM-SLOWFMA-CONTRACT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]]
 
 ; GCN-DENORM-STRICT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; GCN-DENORM-STRICT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]]