AMDGPU: fold fmed3 of fpext sources to f16 fmed3

[arsenm]

Summary

InstCombine already does this for minnum/maxnum. If we
also apply this to fmed3, we don't need to explicitly
use 16-bit fmed3 if we're not sure the target
supports 16-bit instructions yet.

[arsenm]

Rebase on more tests

[foad]

llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp

731

For constants, you need to check that they are exactly representable as half, otherwise this could change the result.

[arsenm]

Check conversion

[foad]

Please add a test case for a constant that can't be losslessly converted. OK with that.

[arsenm]

In D150010#4330382, @foad wrote:

Please add a test case for a constant that can't be losslessly converted. OK with that.

It’s already there

[arsenm]

8f3e64624c2e49b61ee578aec493260a59a35e80

llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp

 #include "AMDGPUTargetTransformInfo.h"
 #include "GCNSubtarget.h"
 #include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <optional>
 
 using namespace llvm;
+using namespace llvm::PatternMatch;
 
 #define DEBUG_TYPE "AMDGPUtti"
 
 namespace {
 
 struct AMDGPUImageDMaskIntrinsic {
   unsigned Intr;
 };
if (isKnownNeverInfOrNaN(Op0, IC.getDataLayout(), TLI, 0,
                            &IC.getAssumptionCache(), &I, &IC.getDominatorTree(),
                            &IC.getOptimizationRemarkEmitter())) {
     // Neither operand is infinity or NaN.
     return true;
   }
   return false;
 }
 
+/// Match an fpext from half to float, or a constant we can convert.
+static bool matchFPExtFromF16(Value *Arg, Value *&FPExtSrc) {
+  if (match(Arg, m_OneUse(m_FPExt(m_Value(FPExtSrc)))))
+    return FPExtSrc->getType()->isHalfTy();
+
+  ConstantFP *CFP;
+  if (match(Arg, m_ConstantFP(CFP))) {
+    bool LosesInfo;
+    APFloat Val(CFP->getValueAPF());
+    Val.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &LosesInfo);
+    if (LosesInfo)
+      return false;
+
+    FPExtSrc = ConstantFP::get(Type::getHalfTy(Arg->getContext()), Val);
+    return true;
+  }
+
+  return false;
+}
+
 std::optional<Instruction *>
 GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
   Intrinsic::ID IID = II.getIntrinsicID();
   switch (IID) {
   case Intrinsic::amdgcn_rcp: {
     Value *Src = II.getArgOperand(0);
 
     // TODO: Move to ConstantFolding/InstSimplify?
if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
           APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(),
                                        C2->getValueAPF());
           return IC.replaceInstUsesWith(
               II, ConstantFP::get(IC.Builder.getContext(), Result));
         }
       }
     }
 
+    if (!ST->hasMed3_16())
+      break;
+
+    Value *X, *Y, *Z;
+
+    // Repeat floating-point width reduction done for minnum/maxnum.
+    // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
+    if (matchFPExtFromF16(Src0, X) && matchFPExtFromF16(Src1, Y) &&
+        matchFPExtFromF16(Src2, Z)) {
+      Value *NewCall = IC.Builder.CreateIntrinsic(IID, {X->getType()},
+                                                  {X, Y, Z}, &II, II.getName());
+      return new FPExtInst(NewCall, II.getType());
+    }
+
     break;
   }
   case Intrinsic::amdgcn_icmp:
   case Intrinsic::amdgcn_fcmp: {
     const ConstantInt *CC = cast<ConstantInt>(II.getArgOperand(2));
     // Guard against invalid arguments.
     int64_t CCVal = CC->getZExtValue();
     bool IsInteger = IID == Intrinsic::amdgcn_icmp;

llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll

 ; NO-FMED3F16-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
 ; NO-FMED3F16-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
 ; NO-FMED3F16-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1:[0-9]+]] {
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[ARG0]], half [[ARG1]], half [[ARG2]])
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg0.ext = fpext half %arg0 to float
   %arg1.ext = fpext half %arg1 to float
   %arg2.ext = fpext half %arg2 to float
   %med3 = call float @llvm.amdgcn.fmed3.f32(float %arg0.ext, float %arg1.ext, float %arg2.ext)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_flags(half %arg0, half %arg1, half %arg2) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_flags
 ; NO-FMED3F16-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
 ; NO-FMED3F16-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
 ; NO-FMED3F16-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call nsz float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_flags
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call nsz float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
+; GFX9-NEXT:    [[MED31:%.*]] = call nsz half @llvm.amdgcn.fmed3.f16(half [[ARG0]], half [[ARG1]], half [[ARG2]])
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg0.ext = fpext half %arg0 to float
   %arg1.ext = fpext half %arg1 to float
   %arg2.ext = fpext half %arg2 to float
   %med3 = call nsz float @llvm.amdgcn.fmed3.f32(float %arg0.ext, float %arg1.ext, float %arg2.ext)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_k0(half %arg1, half %arg2) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_k0
 ; NO-FMED3F16-SAME: (half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
 ; NO-FMED3F16-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG1_EXT]], float [[ARG2_EXT]], float 2.000000e+00)
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_k0
 ; GFX9-SAME: (half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG1_EXT]], float [[ARG2_EXT]], float 2.000000e+00)
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[ARG1]], half [[ARG2]], half 0xH4000)
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg1.ext = fpext half %arg1 to float
   %arg2.ext = fpext half %arg2 to float
   %med3 = call float @llvm.amdgcn.fmed3.f32(float 2.0, float %arg1.ext, float %arg2.ext)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_k1(half %arg0, half %arg2) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_k1
 ; NO-FMED3F16-SAME: (half [[ARG0:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
 ; NO-FMED3F16-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG2_EXT]], float 2.000000e+00)
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_k1
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG2_EXT]], float 2.000000e+00)
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[ARG0]], half [[ARG2]], half 0xH4000)
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg0.ext = fpext half %arg0 to float
   %arg2.ext = fpext half %arg2 to float
   %med3 = call float @llvm.amdgcn.fmed3.f32(float %arg0.ext, float 2.0, float %arg2.ext)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_k2(half %arg0, half %arg1) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_k2
 ; NO-FMED3F16-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
 ; NO-FMED3F16-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float 2.000000e+00)
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_k2
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]]) #[[ATTR1]] {
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[ARG0]] to float
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float 2.000000e+00)
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[ARG0]], half [[ARG1]], half 0xH4000)
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg0.ext = fpext half %arg0 to float
   %arg1.ext = fpext half %arg1 to float
   %med3 = call float @llvm.amdgcn.fmed3.f32(float %arg0.ext, float %arg1.ext, float 2.0)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_k0_k1(half %arg2) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_k0_k1
 ; NO-FMED3F16-SAME: (half [[ARG2:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG2_EXT]], float 0.000000e+00, float 1.600000e+01)
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_k0_k1
 ; GFX9-SAME: (half [[ARG2:%.*]]) #[[ATTR1]] {
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG2_EXT]], float 0.000000e+00, float 1.600000e+01)
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[ARG2]], half 0xH0000, half 0xH4C00)
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg2.ext = fpext half %arg2 to float
   %med3 = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 16.0, float %arg2.ext)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_k0_k2(half %arg1) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_k0_k2
 ; NO-FMED3F16-SAME: (half [[ARG1:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG1_EXT]], float 0.000000e+00, float 2.000000e+00)
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_k0_k2
 ; GFX9-SAME: (half [[ARG1:%.*]]) #[[ATTR1]] {
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[ARG1]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG1_EXT]], float 0.000000e+00, float 2.000000e+00)
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[ARG1]], half 0xH0000, half 0xH4000)
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg1.ext = fpext half %arg1 to float
   %med3 = call float @llvm.amdgcn.fmed3.f32(float 0.0, float %arg1.ext, float 2.0)
   ret float %med3
 }
 
 define float @fmed3_f32_fpext_f16_fabs(half %arg0, half %arg1, half %arg2) #1 {
 ; NO-FMED3F16-LABEL: define float @fmed3_f32_fpext_f16_fabs
 ; NO-FMED3F16-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; NO-FMED3F16-NEXT:    [[FABS_ARG0:%.*]] = call half @llvm.fabs.f16(half [[ARG0]])
 ; NO-FMED3F16-NEXT:    [[FABS_ARG1:%.*]] = call half @llvm.fabs.f16(half [[ARG1]])
 ; NO-FMED3F16-NEXT:    [[FABS_ARG2:%.*]] = call half @llvm.fabs.f16(half [[ARG2]])
 ; NO-FMED3F16-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[FABS_ARG0]] to float
 ; NO-FMED3F16-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[FABS_ARG1]] to float
 ; NO-FMED3F16-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[FABS_ARG2]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_fabs
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; GFX9-NEXT:    [[FABS_ARG0:%.*]] = call half @llvm.fabs.f16(half [[ARG0]])
 ; GFX9-NEXT:    [[FABS_ARG1:%.*]] = call half @llvm.fabs.f16(half [[ARG1]])
 ; GFX9-NEXT:    [[FABS_ARG2:%.*]] = call half @llvm.fabs.f16(half [[ARG2]])
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[FABS_ARG0]] to float
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[FABS_ARG1]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[FABS_ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[FABS_ARG0]], half [[FABS_ARG1]], half [[FABS_ARG2]])
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %fabs.arg0 = call half @llvm.fabs.f16(half %arg0)
   %fabs.arg1 = call half @llvm.fabs.f16(half %arg1)
   %fabs.arg2 = call half @llvm.fabs.f16(half %arg2)
   %arg0.ext = fpext half %fabs.arg0 to float
   %arg1.ext = fpext half %fabs.arg1 to float
   %arg2.ext = fpext half %fabs.arg2 to float

 ; NO-FMED3F16-NEXT:    [[TMP3:%.*]] = call half @llvm.fabs.f16(half [[ARG2]])
 ; NO-FMED3F16-NEXT:    [[FABS_EXT_ARG2:%.*]] = fpext half [[TMP3]] to float
 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[FABS_EXT_ARG0]], float [[FABS_EXT_ARG1]], float [[FABS_EXT_ARG2]])
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_fabs_f32_fpext_f16
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; GFX9-NEXT:    [[TMP1:%.*]] = call half @llvm.fabs.f16(half [[ARG0]])
-; GFX9-NEXT:    [[FABS_EXT_ARG0:%.*]] = fpext half [[TMP1]] to float
 ; GFX9-NEXT:    [[TMP2:%.*]] = call half @llvm.fabs.f16(half [[ARG1]])
-; GFX9-NEXT:    [[FABS_EXT_ARG1:%.*]] = fpext half [[TMP2]] to float
 ; GFX9-NEXT:    [[TMP3:%.*]] = call half @llvm.fabs.f16(half [[ARG2]])
-; GFX9-NEXT:    [[FABS_EXT_ARG2:%.*]] = fpext half [[TMP3]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[FABS_EXT_ARG0]], float [[FABS_EXT_ARG1]], float [[FABS_EXT_ARG2]])
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[TMP1]], half [[TMP2]], half [[TMP3]])
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %arg0.ext = fpext half %arg0 to float
   %arg1.ext = fpext half %arg1 to float
   %arg2.ext = fpext half %arg2 to float
   %fabs.ext.arg0 = call float @llvm.fabs.f32(float %arg0.ext)
   %fabs.ext.arg1 = call float @llvm.fabs.f32(float %arg1.ext)
   %fabs.ext.arg2 = call float @llvm.fabs.f32(float %arg2.ext)

 ; NO-FMED3F16-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
 ; NO-FMED3F16-NEXT:    ret float [[MED3]]
 ;
 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_fneg
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; GFX9-NEXT:    [[FNEG_ARG0:%.*]] = fneg half [[ARG0]]
 ; GFX9-NEXT:    [[FNEG_ARG1:%.*]] = fneg half [[ARG1]]
 ; GFX9-NEXT:    [[FNEG_ARG2:%.*]] = fneg half [[ARG2]]
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[FNEG_ARG0]] to float
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[FNEG_ARG1]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[FNEG_ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[FNEG_ARG0]], half [[FNEG_ARG1]], half [[FNEG_ARG2]])
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %fneg.arg0 = fneg half %arg0
   %fneg.arg1 = fneg half %arg1
   %fneg.arg2 = fneg half %arg2
   %arg0.ext = fpext half %fneg.arg0 to float
   %arg1.ext = fpext half %fneg.arg1 to float
   %arg2.ext = fpext half %fneg.arg2 to float

 ; GFX9-LABEL: define float @fmed3_f32_fpext_f16_fneg_fabs
 ; GFX9-SAME: (half [[ARG0:%.*]], half [[ARG1:%.*]], half [[ARG2:%.*]]) #[[ATTR1]] {
 ; GFX9-NEXT:    [[FABS_ARG0:%.*]] = call half @llvm.fabs.f16(half [[ARG0]])
 ; GFX9-NEXT:    [[FABS_ARG1:%.*]] = call half @llvm.fabs.f16(half [[ARG1]])
 ; GFX9-NEXT:    [[FABS_ARG2:%.*]] = call half @llvm.fabs.f16(half [[ARG2]])
 ; GFX9-NEXT:    [[FNEG_FABS_ARG0:%.*]] = fneg half [[FABS_ARG0]]
 ; GFX9-NEXT:    [[FNEG_FABS_ARG1:%.*]] = fneg half [[FABS_ARG1]]
 ; GFX9-NEXT:    [[FNEG_FABS_ARG2:%.*]] = fneg half [[FABS_ARG2]]
-; GFX9-NEXT:    [[ARG0_EXT:%.*]] = fpext half [[FNEG_FABS_ARG0]] to float
-; GFX9-NEXT:    [[ARG1_EXT:%.*]] = fpext half [[FNEG_FABS_ARG1]] to float
-; GFX9-NEXT:    [[ARG2_EXT:%.*]] = fpext half [[FNEG_FABS_ARG2]] to float
-; GFX9-NEXT:    [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ARG0_EXT]], float [[ARG1_EXT]], float [[ARG2_EXT]])
+; GFX9-NEXT:    [[MED31:%.*]] = call half @llvm.amdgcn.fmed3.f16(half [[FNEG_FABS_ARG0]], half [[FNEG_FABS_ARG1]], half [[FNEG_FABS_ARG2]])
+; GFX9-NEXT:    [[MED3:%.*]] = fpext half [[MED31]] to float
 ; GFX9-NEXT:    ret float [[MED3]]
 ;
   %fabs.arg0 = call half @llvm.fabs.f16(half %arg0)
   %fabs.arg1 = call half @llvm.fabs.f16(half %arg1)
   %fabs.arg2 = call half @llvm.fabs.f16(half %arg2)
   %fneg.fabs.arg0 = fneg half %fabs.arg0
   %fneg.fabs.arg1 = fneg half %fabs.arg1
   %fneg.fabs.arg2 = fneg half %fabs.arg2