AMDGPU: Start selecting v_mad_mix_f32

[rampitec]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1671

What practical sense does it have, if we default to support f32 denorms on GFX9 and have no such instructions before?

[arsenm]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1671

Graphics still disables f32 denormals

[arsenm]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1671

Also the TODO is for the case where we can use this with denormals enabled.

[rampitec]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1671

OK. What prevents FMAD from being selected if denorms are supported?

[arsenm]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1671

We should also probably bring back the less-precise-fpmad option. That would also probably make this valid to use

[rampitec]

LGTM

[arsenm]

r312732

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

private:
   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
   bool SelectSMRDBufferSgpr(SDValue Addr, SDValue &Offset) const;
   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
 
   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
+  bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                        SDValue &Clamp, SDValue &Omod) const;
   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                          SDValue &Clamp, SDValue &Omod) const;
 
   bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
private:
 
   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods,
                         SDValue &Clamp) const;
 
   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                             SDValue &Clamp) const;
+  bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
+  bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 
   void SelectADD_SUB_I64(SDNode *N);
   void SelectUADDO_USUBO(SDNode *N);
   void SelectDIV_SCALE(SDNode *N);
   void SelectFMA_W_CHAIN(SDNode *N);
   void SelectFMUL_W_CHAIN(SDNode *N);
 
   SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
                    uint32_t Offset, uint32_t Width);
   void SelectS_BFEFromShifts(SDNode *N);
   void SelectS_BFE(SDNode *N);
   bool isCBranchSCC(const SDNode *N) const;
   void SelectBRCOND(SDNode *N);
+  void SelectFMAD(SDNode *N);
   void SelectATOMIC_CMP_SWAP(SDNode *N);
 
 protected:
   // Include the pieces autogenerated from the target description.
 #include "AMDGPUGenDAGISel.inc"
 };
 
 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
case ISD::SIGN_EXTEND_INREG:
     if (N->getValueType(0) != MVT::i32)
       break;
 
     SelectS_BFE(N);
     return;
   case ISD::BRCOND:
     SelectBRCOND(N);
     return;
-
+  case ISD::FMAD:
+    SelectFMAD(N);
+    return;
   case AMDGPUISD::ATOMIC_CMP_SWAP:
     SelectATOMIC_CMP_SWAP(N);
     return;
   }
 
   SelectCode(N);
 }
 
void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
   SDLoc SL(N);
 
   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, AMDGPU::VCC, Cond);
   CurDAG->SelectNodeTo(N, AMDGPU::S_CBRANCH_VCCNZ, MVT::Other,
                        N->getOperand(2), // Basic Block
                        VCC.getValue(0));
 }
 
+void AMDGPUDAGToDAGISel::SelectFMAD(SDNode *N) {
+  MVT VT = N->getSimpleValueType(0);
+  if (VT != MVT::f32 || !Subtarget->hasMadMixInsts()) {
+    SelectCode(N);
+    return;
+  }
+
+  SDValue Src0 = N->getOperand(0);
+  SDValue Src1 = N->getOperand(1);
+  SDValue Src2 = N->getOperand(2);
+  unsigned Src0Mods, Src1Mods, Src2Mods;
+
+  // Avoid using v_mad_mix_f32 unless there is actually an operand using the
+  // conversion from f16.
+  bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
+  bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
+  bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
+
+  assert(!Subtarget->hasFP32Denormals() &&
+         "fmad selected with denormals enabled");
+  // TODO: We can select this with f32 denormals enabled if all the sources are
+  // converted from f16 (in which case fmad isn't legal).
+
+  if (Sel0 || Sel1 || Sel2) {
+    // For dummy operands.
+    SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
+    SDValue Ops[] = {
+      CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
+      CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
+      CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
+      CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
+      Zero, Zero
+    };
+
+    CurDAG->SelectNodeTo(N, AMDGPU::V_MAD_MIX_F32, MVT::f32, Ops);
+  } else {
+    SelectCode(N);
+  }
+}
+
 // This is here because there isn't a way to use the generated sub0_sub1 as the
 // subreg index to EXTRACT_SUBREG in tablegen.
 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
   MemSDNode *Mem = cast<MemSDNode>(N);
   unsigned AS = Mem->getAddressSpace();
   if (AS == AMDGPUASI.FLAT_ADDRESS) {
     SelectCode(N);
     return;
void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
   SDValue Extract
     = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
 
   ReplaceUses(SDValue(N, 0), Extract);
   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
   CurDAG->RemoveDeadNode(N);
 }
 
-bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
-                                        SDValue &SrcMods) const {
-  unsigned Mods = 0;
+bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
+                                            unsigned &Mods) const {
+  Mods = 0;
   Src = In;
 
   if (Src.getOpcode() == ISD::FNEG) {
     Mods |= SISrcMods::NEG;
     Src = Src.getOperand(0);
   }
 
   if (Src.getOpcode() == ISD::FABS) {
     Mods |= SISrcMods::ABS;
     Src = Src.getOperand(0);
   }
 
+  return true;
+}
+
+bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
+                                        SDValue &SrcMods) const {
+  unsigned Mods;
+  if (SelectVOP3ModsImpl(In, Src, Mods)) {
     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
     return true;
   }
 
+  return false;
+}
+
 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
                                              SDValue &SrcMods) const {
   SelectVOP3Mods(In, Src, SrcMods);
   return isNoNanSrc(Src);
 }
 
 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src,
   SDLoc SL(In);
 
   // FIXME: Handle clamp
   Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
 
   return SelectVOP3OpSelMods(In, Src, SrcMods);
 }
 
+// The return value is not whether the match is possible (which it always is),
+// but whether or not it a conversion is really used.
+bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
+                                                   unsigned &Mods) const {
+  Mods = 0;
+  SelectVOP3ModsImpl(In, Src, Mods);
+
+  if (Src.getOpcode() == ISD::FP_EXTEND) {
+    Src = Src.getOperand(0);
+    assert(Src.getValueType() == MVT::f16);
+    Src = stripBitcast(Src);
+
+    // op_sel/op_sel_hi decide the source type and source.
+    // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
+    // If the sources's op_sel is set, it picks the high half of the source
+    // register.
+
+    Mods |= SISrcMods::OP_SEL_1;
+    if (isExtractHiElt(Src, Src))
+      Mods |= SISrcMods::OP_SEL_0;
+
+    return true;
+  }
+
+  return false;
+}
+
+bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
+                                               SDValue &SrcMods) const {
+  unsigned Mods = 0;
+  SelectVOP3PMadMixModsImpl(In, Src, Mods);
+  SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
+  return true;
+}
+
 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
   const AMDGPUTargetLowering& Lowering =
     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
   bool IsModified = false;
   do {
     IsModified = false;
     // Go over all selected nodes and try to fold them a bit more
     for (SDNode &Node : CurDAG->allnodes()) {

lib/Target/AMDGPU/AMDGPUSubtarget.h

public:
   bool hasMed3_16() const {
     return getGeneration() >= GFX9;
   }
 
   bool hasMin3Max3_16() const {
     return getGeneration() >= GFX9;
   }
 
+  bool hasMadMixInsts() const {
+    return getGeneration() >= GFX9;
+  }
+
   bool hasCARRY() const {
     return (getGeneration() >= EVERGREEN);
   }
 
   bool hasBORROW() const {
     return (getGeneration() >= EVERGREEN);
   }
 

lib/Target/AMDGPU/SIInstrInfo.td

 def VOP3PMods0 : ComplexPattern<untyped, 3, "SelectVOP3PMods0">;
 
 def VOP3OpSel  : ComplexPattern<untyped, 2, "SelectVOP3OpSel">;
 def VOP3OpSel0 : ComplexPattern<untyped, 3, "SelectVOP3OpSel0">;
 
 def VOP3OpSelMods  : ComplexPattern<untyped, 2, "SelectVOP3OpSelMods">;
 def VOP3OpSelMods0 : ComplexPattern<untyped, 3, "SelectVOP3OpSelMods0">;
 
+def VOP3PMadMixMods  : ComplexPattern<untyped, 2, "SelectVOP3PMadMixMods">;
+
 //===----------------------------------------------------------------------===//
 // SI assembler operands
 //===----------------------------------------------------------------------===//
 
 def SIOperand {
   int ZERO = 0x80;
   int VCC = 0x6A;
   int FLAT_SCR = 0x68;

test/CodeGen/AMDGPU/mad-mix.ll

+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2{{$}}
+; VI: v_mac_f32
+; CI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16hi_f16hi_f16hi_int:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1]{{$}}
+; CIVI: v_mac_f32
+define float @v_mad_mix_f32_f16hi_f16hi_f16hi_int(i32 %src0, i32 %src1, i32 %src2) #0 {
+  %src0.hi = lshr i32 %src0, 16
+  %src1.hi = lshr i32 %src1, 16
+  %src2.hi = lshr i32 %src2, 16
+  %src0.i16 = trunc i32 %src0.hi to i16
+  %src1.i16 = trunc i32 %src1.hi to i16
+  %src2.i16 = trunc i32 %src2.hi to i16
+  %src0.fp16 = bitcast i16 %src0.i16 to half
+  %src1.fp16 = bitcast i16 %src1.i16 to half
+  %src2.fp16 = bitcast i16 %src2.i16 to half
+  %src0.ext = fpext half %src0.fp16 to float
+  %src1.ext = fpext half %src1.fp16 to float
+  %src2.ext = fpext half %src2.fp16 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16hi_f16hi_f16hi_elt:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1]{{$}}
+; VI: v_mac_f32
+; CI: v_mad_f32
+define float @v_mad_mix_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.hi = extractelement <2 x half> %src0, i32 1
+  %src1.hi = extractelement <2 x half> %src1, i32 1
+  %src2.hi = extractelement <2 x half> %src2, i32 1
+  %src0.ext = fpext half %src0.hi to float
+  %src1.ext = fpext half %src1.hi to float
+  %src2.ext = fpext half %src2.hi to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32:
+; GFX9: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mad_mix_f32 v1, v0, v3, v2 op_sel:[1,1,1]
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v3, v2
+
+; CIVI: v_mac_f32
+define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_shuffle:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mad_mix_f32 v1, v0, v3, v2 op_sel:[0,1,1]
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v3, v2 op_sel:[1,0,1]
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mac_f32
+define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.shuf = shufflevector <2 x half> %src0, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %src1.shuf = shufflevector <2 x half> %src1, <2 x half> undef, <2 x i32> <i32 0, i32 1>
+  %src2.shuf = shufflevector <2 x half> %src2, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %src0.ext = fpext <2 x half> %src0.shuf to <2 x float>
+  %src1.ext = fpext <2 x half> %src1.shuf to <2 x float>
+  %src2.ext = fpext <2 x half> %src2.shuf to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_negf16lo_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_negf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %src0.ext.neg = fsub float -0.0, %src0.ext
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.neg, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_absf16lo_f16lo_f16lo:
+; GFX9: v_mad_mix_f32 v0, |v0|, v1, v2
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_absf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.abs, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_negabsf16lo_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -|v0|, v1, v2
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_negabsf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
+  %src0.ext.neg.abs = fsub float -0.0, %src0.ext.abs
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.neg.abs, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_negf32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_negf32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.neg = fsub float -0.0, %src2
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.neg)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_absf32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, |v2| op_sel_hi:[1,1,0]{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_absf32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.abs = call float @llvm.fabs.f32(float %src2)
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.abs)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_negabsf32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, -|v2| op_sel_hi:[1,1,0]{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_negabsf32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.abs = call float @llvm.fabs.f32(float %src2)
+  %src2.neg.abs = fsub float -0.0, %src2.abs
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.neg.abs)
+  ret float %result
+}
+
+; TODO: Fold inline immediates. Need to be careful because it is an
+; f16 inline immediate that may be converted to f32, not an actual f32
+; inline immediate.
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; GCN: s_waitcnt
+; GFX9: v_mov_b32_e32 v2, 1.0
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+
+; CIVI: v_mad_f32 v0, v0, v1, 1.0
+; GCN-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_f16lo_f16lo_f32imm1(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 1.0)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; GCN: s_waitcnt
+; GFX9: v_mov_b32_e32 v2, 0.15915494
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+; VI: v_mad_f32 v0, v0, v1, 0.15915494
+define float @v_mad_mix_f32_f16lo_f16lo_f32imminv2pi(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 0x3FC45F3060000000)
+  ret float %result
+}
+
+; Attempt to break inline immediate folding. If the operand is
+; interpreted as f32, the inline immediate is really the f16 inline
+; imm value converted to f32.
+; fpext f16 1/2pi = 0x3e230000
+;       f32 1/2pi = 0x3e22f983
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GFX9: v_mov_b32_e32 v2, 0x3e230000
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+
+; CIVI: v_madak_f32 v0, v0, v1, 0x3e230000
+define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2 = fpext half 0xH3118 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GFX9: v_mov_b32_e32 v2, 0x367c0000
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+
+; CIVI: v_madak_f32 v0, v0, v1, 0x367c0000
+define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imm63(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2 = fpext half 0xH003F to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_f32imm1:
+; GFX9: v_mov_b32_e32 v2, v1
+; GFX9: v_mov_b32_e32 v3, 1.0
+; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]{{$}}
+; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0]{{$}}
+define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 1.0, float 1.0>)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_cvtf16imminv2pi:
+; GFX9: v_mov_b32_e32 v2, v1
+; GFX9: v_mov_b32_e32 v3, 0x3e230000
+; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]{{$}}
+; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0]{{$}}
+define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_f32imminv2pi:
+; GFX9: v_mov_b32_e32 v2, v1
+; GFX9: v_mov_b32_e32 v3, 0.15915494
+; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]{{$}}
+; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0]{{$}}
+define <2 x float> @v_mad_mix_v2f32_f32imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 0x3FC45F3060000000, float 0x3FC45F3060000000>)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] clamp{{$}}
+; VI: v_mac_f32_e64 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
+; CI: v_mad_f32 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
+define float @v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.hi = extractelement <2 x half> %src0, i32 1
+  %src1.hi = extractelement <2 x half> %src1, i32 1
+  %src2.hi = extractelement <2 x half> %src2, i32 1
+  %src0.ext = fpext half %src0.hi to float
+  %src1.ext = fpext half %src1.hi to float
+  %src2.ext = fpext half %src2.hi to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %max = call float @llvm.maxnum.f32(float %result, float 0.0)
+  %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
+  ret float %clamp
+}
+
+; GCN-LABEL: no_mix_simple:
+; GCN: s_waitcnt
+; GCN-NEXT: v_mad_f32 v0, v0, v1, v2{{$}}
+; GCN-NEXT: s_setpc_b64
+define float @no_mix_simple(float %src0, float %src1, float %src2) #0 {
+  %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: no_mix_simple_fabs:
+; GCN: s_waitcnt
+; GCN-NEXT: v_mad_f32 v0, |v0|, v1, v2{{$}}
+; GCN-NEXT: s_setpc_b64
+define float @no_mix_simple_fabs(float %src0, float %src1, float %src2) #0 {
+  %src0.fabs = call float @llvm.fabs.f32(float %src0)
+  %result = call float @llvm.fmuladd.f32(float %src0.fabs, float %src1, float %src2)
+  ret float %result
+}
+
+; FIXME: Should abe able to select in thits case
+; All sources are converted from f16, so it doesn't matter
+; v_mad_mix_f32 flushes.
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_fma_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals(half %src0, half %src1, half %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_denormals:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_fma_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals(half %src0, half %src1, float %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_mul_f32
+; GFX9: v_add_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, half %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2.ext
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_mul_f32
+; GFX9: v_add_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, float %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_flush_fmulfadd:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2{{$}}
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2.ext
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2
+  ret float %result
+}
+
+declare float @llvm.fabs.f32(float) #2
+declare float @llvm.minnum.f32(float, float) #2
+declare float @llvm.maxnum.f32(float, float) #2
+declare float @llvm.fmuladd.f32(float, float, float) #2
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #2
+
+attributes #0 = { nounwind "target-features"="-fp32-denormals" }
+attributes #1 = { nounwind "target-features"="+fp32-denormals" }
+attributes #2 = { nounwind readnone speculatable }