[AMDGPU][GlobalISel] Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)

[matejam]

Summary

Instead of add and mul instructions, use v_mad, v_mac or v_fma if fma
instructions are faster and are legal for the given architecture.
Combiner for a case that adds two function arguments, extends the result
and multiplies it with the third function argument, and transforms that
into extend instructions combined with some of the fma instruction
depending on the architecture.

[matejam]

Put back the accidentally deleted combiner from the list of combiners (load_or_combine).

[matejam]

Minor changes in CombinerHelper.cpp and in the tests.

[matejam]

Use mi_match for comparing instructions instead of comparing the opcodes.

[matejam]

A few minor changes (removal of unnecessary comments).

[matejam]

Rebased.

[matejam]

Use m_MInstr instead of m_Reg in matching patterns (mi_match).
Formatting and refactoring.

[matejam]

Use applyBuildFn instead of writing my own apply.

[mbrkusanin]

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

4929–4935

Use B not Builder.
Also maybe rename from FpExt1 and FpExt2 to FpExtX and FpExtY so it matches the comment above.

4948–4955

(same as above)
Use B not Builder.
Also maybe rename from FpExt1 and FpExt2 to FpExtY and FpExtZ so it matches the comment above.

[matejam]

Changes in tests.

[mbrkusanin]

LGTM

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

382–383

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

4938

Operand names in this comment don't match variables used below (FpExtX, FpExtY). Either update comment or rename variables below.

[matejam]

Added missing comments and an FMA combiner group in table gen.

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

public:
 
   /// Match fabs(fabs(x)) to fabs(x).
   bool matchCombineFAbsOfFAbs(MachineInstr &MI, Register &Src);
   void applyCombineFAbsOfFAbs(MachineInstr &MI, Register &Src);
 
   /// Transform fabs(fneg(x)) to fabs(x).
   bool matchCombineFAbsOfFNeg(MachineInstr &MI, BuildFnTy &MatchInfo);
 
   /// Transform trunc ([asz]ext x) to x or ([asz]ext x) or (trunc x).
   bool matchCombineTruncOfExt(MachineInstr &MI,
                               std::pair<Register, unsigned> &MatchInfo);
   void applyCombineTruncOfExt(MachineInstr &MI,
                               std::pair<Register, unsigned> &MatchInfo);
 
   /// Transform trunc (shl x, K) to shl (trunc x),
   /// K => K < VT.getScalarSizeInBits().
   bool matchCombineTruncOfShl(MachineInstr &MI,
                               std::pair<Register, Register> &MatchInfo);
public:
 
   bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally,
                            bool &HasFMAD, bool &Aggressive);
 
   /// Transform (fadd (fmul x, y), z) -> (fma x, y, z)
   ///           (fadd (fmul x, y), z) -> (fmad x, y, z)
   bool matchCombineFAddFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo);
 
+  /// Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
+  ///           (fadd (fpext (fmul x, y)), z) -> (fmad (fpext x), (fpext y), z)
+  bool matchCombineFAddFpExtFMulToFMadOrFMA(MachineInstr &MI,
+                                            BuildFnTy &MatchInfo);
+
 private:
   /// Given a non-indexed load or store instruction \p MI, find an offset that
   /// can be usefully and legally folded into it as a post-indexing operation.
   ///
   /// \returns true if a candidate is found.
   bool findPostIndexCandidate(MachineInstr &MI, Register &Addr, Register &Base,
                               Register &Offset);
 

llvm/include/llvm/CodeGen/TargetLowering.h

virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const {
     assert(SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() &&
            "invalid fpext types");
     return false;
   }
 
   /// Return true if an fpext operation input to an \p Opcode operation is free
   /// (for instance, because half-precision floating-point numbers are
   /// implicitly extended to float-precision) for an FMA instruction.
+  virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
+                               LLT DestTy, LLT SrcTy) const {
+    return false;
+  }
+
+  /// Return true if an fpext operation input to an \p Opcode operation is free
+  /// (for instance, because half-precision floating-point numbers are
+  /// implicitly extended to float-precision) for an FMA instruction.
   virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
                                EVT DestVT, EVT SrcVT) const {
     assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
            "invalid fpext types");
     return isFPExtFree(DestVT, SrcVT);
   }
 
   /// Return true if folding a vector load into ExtVal (a sign, zero, or any

llvm/include/llvm/Target/GlobalISel/Combine.td

 //           (fadd (fmul x, y), z) -> (fmad x, y, z)
 def combine_fadd_fmul_to_fmad_or_fma: GICombineRule<
   (defs root:$root, build_fn_matchinfo:$info),
   (match (wip_match_opcode G_FADD):$root,
          [{ return Helper.matchCombineFAddFMulToFMadOrFMA(*${root},
                                                           ${info}); }]),
   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
 
+// Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
+//                                         -> (fmad (fpext x), (fpext y), z)
+// Transform (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
+//                                         -> (fmad (fpext y), (fpext z), x)
+def combine_fadd_fpext_fmul_to_fmad_or_fma: GICombineRule<
+  (defs root:$root, build_fn_matchinfo:$info),
+  (match (wip_match_opcode G_FADD):$root,
+         [{ return Helper.matchCombineFAddFpExtFMulToFMadOrFMA(*${root},
+                                                               ${info}); }]),
+  (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
 
 // FIXME: These should use the custom predicate feature once it lands.
 def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      undef_to_negative_one,
                                      binop_left_undef_to_zero,
                                      propagate_undef_any_op,
                                      propagate_undef_all_ops,
                                      propagate_undef_shuffle_mask,

 
 def phi_combines : GICombineGroup<[extend_through_phis]>;
 
 def select_combines : GICombineGroup<[select_undef_cmp, select_constant_cmp]>;
 
 def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl, add_p2i_to_ptradd,
                                        mul_by_neg_one]>;
 
-def fma_combines : GICombineGroup<[combine_fadd_fmul_to_fmad_or_fma]>;
+def fma_combines : GICombineGroup<[combine_fadd_fmul_to_fmad_or_fma,
+  combine_fadd_fpext_fmul_to_fmad_or_fma]>;
 
 def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines,
     extract_vec_elt_combines, combines_for_extload,
     combine_indexed_load_store, undef_combines, identity_combines, phi_combines,
     simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands,
     reassocs, ptr_add_immed_chain,
     shl_ashr_to_sext_inreg, sext_inreg_of_load,
     width_reduction_combines, select_combines,

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

MatchInfo = [=, &MI](MachineIRBuilder &B) {
                     LHS->getOperand(0).getReg()});
     };
     return true;
   }
 
   return false;
 }
 
+bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(
+    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) {
+  assert(MI.getOpcode() == TargetOpcode::G_FADD);
+
+  bool AllowFusionGlobally, HasFMAD, Aggressive;
+  if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
+    return false;
+
+  const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
+  MachineInstr *LHS = MRI.getVRegDef(MI.getOperand(1).getReg());
+  MachineInstr *RHS = MRI.getVRegDef(MI.getOperand(2).getReg());
+  LLT DstType = MRI.getType(MI.getOperand(0).getReg());
+
+  unsigned PreferredFusedOpcode =
+      HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
+
+  // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
+  // prefer to fold the multiply with fewer uses.
+  if (Aggressive && isContractableFMul(*LHS, AllowFusionGlobally) &&
+      isContractableFMul(*RHS, AllowFusionGlobally)) {
+    if (hasMoreUses(*LHS, *RHS, MRI))
+      std::swap(LHS, RHS);
+  }
+
+  // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
+  MachineInstr *FpExtSrc;
+  if (mi_match(LHS->getOperand(0).getReg(), MRI,
+               m_GFPExt(m_MInstr(FpExtSrc))) &&
+      isContractableFMul(*FpExtSrc, AllowFusionGlobally) &&
+      TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
+                          MRI.getType(FpExtSrc->getOperand(1).getReg()))) {
+    MatchInfo = [=, &MI](MachineIRBuilder &B) {
+      auto FpExtX = B.buildFPExt(DstType, FpExtSrc->getOperand(1).getReg());
+      auto FpExtY = B.buildFPExt(DstType, FpExtSrc->getOperand(2).getReg());
+      B.buildInstr(
+          PreferredFusedOpcode, {MI.getOperand(0).getReg()},
+          {FpExtX.getReg(0), FpExtY.getReg(0), RHS->getOperand(0).getReg()});
+    };
+    return true;
+  }
+
+  // fold (fadd z, (fpext (fmul x, y))) -> (fma (fpext x), (fpext y), z)
+  // Note: Commutes FADD operands.
+  if (mi_match(RHS->getOperand(0).getReg(), MRI,
+               m_GFPExt(m_MInstr(FpExtSrc))) &&
+      isContractableFMul(*FpExtSrc, AllowFusionGlobally) &&
+      TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
+                          MRI.getType(FpExtSrc->getOperand(1).getReg()))) {
+    MatchInfo = [=, &MI](MachineIRBuilder &B) {
+      auto FpExtX = B.buildFPExt(DstType, FpExtSrc->getOperand(1).getReg());
+      auto FpExtY = B.buildFPExt(DstType, FpExtSrc->getOperand(2).getReg());
+      B.buildInstr(
+          PreferredFusedOpcode, {MI.getOperand(0).getReg()},
+          {FpExtX.getReg(0), FpExtY.getReg(0), LHS->getOperand(0).getReg()});
+    };
+    return true;
+  }
+
+  return false;
+}
+
 bool CombinerHelper::tryCombine(MachineInstr &MI) {
   if (tryCombineCopy(MI))
     return true;
   if (tryCombineExtendingLoads(MI))
     return true;
   if (tryCombineIndexedLoadStore(MI))
     return true;
   return false;
 }

llvm/lib/Target/AMDGPU/SIISelLowering.h

 public:
   SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI);
 
   const GCNSubtarget *getSubtarget() const;
 
   bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT,
                        EVT SrcVT) const override;
 
+  bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy,
+                       LLT SrcTy) const override;
+
   bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override;
 
   bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
                           MachineFunction &MF,
                           unsigned IntrinsicID) const override;
 
   bool getAddrModeArguments(IntrinsicInst * /*I*/,
                             SmallVectorImpl<Value*> &/*Ops*/,

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
   return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
           (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
     DestVT.getScalarType() == MVT::f32 &&
     SrcVT.getScalarType() == MVT::f16 &&
     // TODO: This probably only requires no input flushing?
     !hasFP32Denormals(DAG.getMachineFunction());
 }
 
+bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
+                                       LLT DestTy, LLT SrcTy) const {
+  return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
+          (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
+         DestTy.getScalarSizeInBits() == 32 &&
+         SrcTy.getScalarSizeInBits() == 16 &&
+         // TODO: This probably only requires no input flushing?
+         !hasFP32Denormals(*MI.getMF());
+}
+
 bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
   // SI has some legal vector types, but no legal vector operations. Say no
   // shuffles are legal in order to prefer scalarizing some vector operations.
   return false;
 }
 
 MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                     CallingConv::ID CC,

llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-mul.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-FAST-DENORM %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-FAST-DENORM %s
+
+; fold (fadd fast (fpext (fmul fast x, y)), z) -> (fma (fpext x), (fpext y), z)
+; fold (fadd fast x, (fpext (fmul fast y, z))) -> (fma (fpext y), (fpext z), x)
+
+define amdgpu_vs float @test_f16_f32_add_ext_mul(half inreg %x, half inreg %y, float inreg %z) {
+; GFX9-FAST-DENORM-LABEL: test_f16_f32_add_ext_mul:
+; GFX9-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v1, s1
+; GFX9-FAST-DENORM-NEXT:    v_mad_f32 v0, v0, v1, s2
+; GFX9-FAST-DENORM-NEXT:    ; return to shader part epilog
+;
+; GFX10-FAST-DENORM-LABEL: test_f16_f32_add_ext_mul:
+; GFX10-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v1, s1
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v0, v0, v1, s2
+; GFX10-FAST-DENORM-NEXT:    ; return to shader part epilog
+.entry:
+    %a = fmul fast half %x, %y
+    %b = fpext half %a to float
+    %c = fadd fast float %b, %z
+    ret float %c
+}
+
+define amdgpu_vs float @test_f16_f32_add_ext_mul_rhs(half inreg %x, half inreg %y, float inreg %z) {
+; GFX9-FAST-DENORM-LABEL: test_f16_f32_add_ext_mul_rhs:
+; GFX9-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v1, s1
+; GFX9-FAST-DENORM-NEXT:    v_mad_f32 v0, v0, v1, s2
+; GFX9-FAST-DENORM-NEXT:    ; return to shader part epilog
+;
+; GFX10-FAST-DENORM-LABEL: test_f16_f32_add_ext_mul_rhs:
+; GFX10-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v1, s1
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v0, v0, v1, s2
+; GFX10-FAST-DENORM-NEXT:    ; return to shader part epilog
+.entry:
+    %a = fmul fast half %x, %y
+    %b = fpext half %a to float
+    %c = fadd fast float %z, %b
+    ret float %c
+}
+
+define amdgpu_vs <5 x float> @test_5xf16_5xf32_add_ext_mul(<5 x half> inreg %x, <5 x half> inreg %y, <5 x float> inreg %z) {
+; GFX9-FAST-DENORM-LABEL: test_5xf16_5xf32_add_ext_mul:
+; GFX9-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX9-FAST-DENORM-NEXT:    s_pack_lh_b32_b16 s3, s3, s3
+; GFX9-FAST-DENORM-NEXT:    s_pack_lh_b32_b16 s4, s4, s4
+; GFX9-FAST-DENORM-NEXT:    s_pack_lh_b32_b16 s0, s0, s0
+; GFX9-FAST-DENORM-NEXT:    s_pack_lh_b32_b16 s1, s1, s1
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v0, s0, v0
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v1, s1, v1
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v2, s2, v2
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, v1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v7, v2
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s6, v3
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v1, s7, v4
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v2, s8, v5
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v3, s9, v6
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v4, s10, v7
+; GFX9-FAST-DENORM-NEXT:    ; return to shader part epilog
+;
+; GFX10-FAST-DENORM-LABEL: test_5xf16_5xf32_add_ext_mul:
+; GFX10-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s11, s0, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s12, s1, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s13, s3, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s14, s4, 16
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v1, s11
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v2, s1
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v3, s12
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v4, s2
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, s3
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v6, s13
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v7, s4
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v8, s14
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v9, s5
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v0, v0, v5, s6
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v1, v1, v6, s7
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v2, v2, v7, s8
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v3, v3, v8, s9
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v4, v4, v9, s10
+; GFX10-FAST-DENORM-NEXT:    ; return to shader part epilog
+.entry:
+    %a = fmul fast <5 x half> %x, %y
+    %b = fpext <5 x half> %a to <5 x float>
+    %c = fadd fast <5 x float> %b, %z
+    ret <5 x float> %c
+}
+
+define amdgpu_vs <6 x float> @test_6xf16_6xf32_add_ext_mul_rhs(<6 x half> inreg %x, <6 x half> inreg %y, <6 x float> inreg %z) {
+; GFX9-FAST-DENORM-LABEL: test_6xf16_6xf32_add_ext_mul_rhs:
+; GFX9-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v0, s0, v0
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v1, s1, v1
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v2, s2, v2
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, v1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v7, v2
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s6, v3
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v1, s7, v4
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v2, s8, v5
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v3, s9, v6
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v4, s10, v7
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v5, s11, v8
+; GFX9-FAST-DENORM-NEXT:    ; return to shader part epilog
+;
+; GFX10-FAST-DENORM-LABEL: test_6xf16_6xf32_add_ext_mul_rhs:
+; GFX10-FAST-DENORM:       ; %bb.0: ; %.entry
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s12, s0, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s13, s1, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s14, s2, 16
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v2, s1
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v4, s2
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s0, s3, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s1, s4, 16
+; GFX10-FAST-DENORM-NEXT:    s_lshr_b32 s2, s5, 16
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v1, s12
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v3, s13
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, s14
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v6, s3
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v7, s0
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v8, s4
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v9, s1
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v10, s5
+; GFX10-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v11, s2
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v0, v0, v6, s6
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v1, v1, v7, s7
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v2, v2, v8, s8
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v3, v3, v9, s9
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v4, v4, v10, s10
+; GFX10-FAST-DENORM-NEXT:    v_fma_f32 v5, v5, v11, s11
+; GFX10-FAST-DENORM-NEXT:    ; return to shader part epilog
+.entry:
+    %a = fmul fast <6 x half> %x, %y
+    %b = fpext <6 x half> %a to <6 x float>
+    %c = fadd fast <6 x float> %z, %b
+    ret <6 x float> %c
+}