AMDGPU: Look for src mods before fp_extend

[arsenm]

Summary

When selecting modifiers for mad_mix instructions,
look at fneg/fabs that occur before the conversion.

[rampitec]

LGTM

[arsenm]

r315748

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
   Mods = 0;
   SelectVOP3ModsImpl(In, Src, Mods);
 
   if (Src.getOpcode() == ISD::FP_EXTEND) {
     Src = Src.getOperand(0);
     assert(Src.getValueType() == MVT::f16);
     Src = stripBitcast(Src);
 
+    // Be careful about folding modifiers if we already have an abs. fneg is
+    // applied last, so we don't want to apply an earlier fneg.
+    if ((Mods & SISrcMods::ABS) == 0) {
+      unsigned ModsTmp;
+      SelectVOP3ModsImpl(Src, Src, ModsTmp);
+
+      if ((ModsTmp & SISrcMods::NEG) != 0)
+        Mods ^= SISrcMods::NEG;
+
+      if ((ModsTmp & SISrcMods::ABS) != 0)
+        Mods |= SISrcMods::ABS;
+    }
+
     // op_sel/op_sel_hi decide the source type and source.
     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
     // If the sources's op_sel is set, it picks the high half of the source
     // register.
 
     Mods |= SISrcMods::OP_SEL_1;
-    if (isExtractHiElt(Src, Src))
+    if (isExtractHiElt(Src, Src)) {
       Mods |= SISrcMods::OP_SEL_0;
 
+      // TODO: Should we try to look for neg/abs here?
+    }
+
     return true;
   }
 
   return false;
 }
 
 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
                                                SDValue &SrcMods) const {

test/CodeGen/AMDGPU/fpext-free.ll

entry:
   %mul.ext = fpext half %mul to float
   %add = fsub contract float %x, %mul.ext
   ret float %add
 }
 
 ; fold (fsub (fpext (fneg (fmul, x, y))), z)
 ;   -> (fneg (fma (fpext x), (fpext y), z))
 
-; FIXME: Should be able to fold fneg
 ; GCN-LABEL: {{^}}fsub_fpext_fneg_fmul_f16_to_f32:
 ; GCN: s_waitcnt
-; GFX9-F32FLUSH-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]{{$}}
 ; GFX9-F32FLUSH-NEXT: s_setpc_b64
 
 ; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
 ; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
 ; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
 ; GFX9-F32DENORM-NEXT: s_setpc_b64
 define float @fsub_fpext_fneg_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
 entry:
   %mul = fmul half %x, %y
   %neg.mul = fsub half -0.0, %mul
   %neg.mul.ext = fpext half %neg.mul to float
   %add = fsub float %neg.mul.ext, %z
   ret float %add
 }
 
 ; fold (fsub (fneg (fpext (fmul, x, y))), z)
 ;   -> (fneg (fma (fpext x)), (fpext y), z)
 
-; FIXME: Should be able to fold fneg
 ; GCN-LABEL: {{^}}fsub_fneg_fpext_fmul_f16_to_f32:
 ; GCN: s_waitcnt
-; GFX9-F32FLUSH-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]{{$}}
 ; GFX9-F32FLUSH-NEXT: s_setpc_b64
 
 ; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
 ; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
 ; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
 ; GFX9-F32DENORM-NEXT: s_setpc_b64
 define float @fsub_fneg_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
 entry:

test/CodeGen/AMDGPU/mad-mix.ll

 define float @v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, float %src2) #0 {
   %src0.ext = fpext half %src0 to float
   %src1.ext = fpext half %src1 to float
   %mul = fmul float %src0.ext, %src1.ext
   %result = fadd float %mul, %src2
   ret float %result
 }
 
+; GCN-LABEL: {{^}}v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %src0 = extractelement <2 x half> %src0.arg.bc, i32 0
+  %src0.neg = fsub half -0.0, %src0
+  %src0.ext = fpext half %src0.neg to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+;  %src0.ext.neg = fsub float -0.0, %src0.ext
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; Make sure we don't fold pre-cvt fneg if we already have a fabs
+; GCN-LABEL: {{^}}v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; GFX9: s_waitcnt
+define float @v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %src0 = extractelement <2 x half> %src0.arg.bc, i32 1
+  %src0.neg = fsub half -0.0, %src0
+  %src0.ext = fpext half %src0.neg to float
+  %src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.abs, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_precvtabsf16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, |v0|, v1, v2 op_sel:[1,0,0]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_precvtabsf16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %src0 = extractelement <2 x half> %src0.arg.bc, i32 1
+  %src0.abs = call half @llvm.fabs.f16(half %src0)
+  %src0.ext = fpext half %src0.abs to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 op_sel:[1,0,0]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %fneg = fsub <2 x half> <half -0.0, half -0.0>, %src0.arg.bc
+  %src0 = extractelement <2 x half> %fneg, i32 1
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; FIXME: Should be able to fold
+; GCN-LABEL: {{^}}v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,0,0]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
+  %src0 = extractelement <2 x half> %fabs, i32 1
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; FIXME: Should be able to fold
+; GCN-LABEL: {{^}}v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 op_sel:[1,0,0]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
+  %fneg.fabs = fsub <2 x half> <half -0.0, half -0.0>, %fabs
+  %src0 = extractelement <2 x half> %fneg.fabs, i32 1
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+declare half @llvm.fabs.f16(half) #2
+declare <2 x half> @llvm.fabs.v2f16(<2 x half>) #2
 declare float @llvm.fabs.f32(float) #2
 declare float @llvm.minnum.f32(float, float) #2
 declare float @llvm.maxnum.f32(float, float) #2
 declare float @llvm.fmuladd.f32(float, float, float) #2
 declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #2
 
 attributes #0 = { nounwind "target-features"="-fp32-denormals" }
 attributes #1 = { nounwind "target-features"="+fp32-denormals" }
 attributes #2 = { nounwind readnone speculatable }