AMDGPU: Use clamp with f64

[artem.tamazov]

LGTM

[arsenm]

r295908

lib/Target/AMDGPU/SIFoldOperands.cpp

for (FoldCandidate &Fold : FoldList) {
     }
   }
 }
 
 const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const {
   unsigned Op = MI.getOpcode();
   switch (Op) {
   case AMDGPU::V_MAX_F32_e64:
-  case AMDGPU::V_MAX_F16_e64: {
+  case AMDGPU::V_MAX_F16_e64:
+  case AMDGPU::V_MAX_F64: {
     if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
       return nullptr;
 
     // Make sure sources are identical.
     const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
     const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
     if (!Src0->isReg() || Src0->getSubReg() != Src1->getSubReg() ||
         Src0->getSubReg() != AMDGPU::NoSubRegister)

lib/Target/AMDGPU/SIISelLowering.cpp

if (Subtarget->enableDX10Clamp()) {
     // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
     // hardware fmed3 behavior converting to a min.
     // FIXME: Should this be allowing -0.0?
     if (K1->isExactlyValue(1.0) && K0->isExactlyValue(0.0))
       return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0));
   }
 
   // No med3 for f16, but clamp is possible.
-  if (VT == MVT::f16)
+  // TODO: gfx9 has med3 f16
+  if (VT == MVT::f16 || VT == MVT::f64)
     return SDValue();
 
   // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
   // signaling NaN gives a quiet NaN. The quiet NaN input to the min would then
   // give the other result, which is different from med3 with a NaN input.
   SDValue Var = Op0.getOperand(0);
   if (!isKnownNeverSNan(DAG, Var))
     return SDValue();
 
   return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0),
                      Var, SDValue(K0, 0), SDValue(K1, 0));
 }
 
 SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
 
+  EVT VT = N->getValueType(0);
   unsigned Opc = N->getOpcode();
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
 
   // Only do this if the inner op has one use since this will just increases
   // register pressure for no benefit.
 
-  if (Opc != AMDGPUISD::FMIN_LEGACY && Opc != AMDGPUISD::FMAX_LEGACY) {
+
+  if (Opc != AMDGPUISD::FMIN_LEGACY && Opc != AMDGPUISD::FMAX_LEGACY &&
+      VT != MVT::f64) {
     // max(max(a, b), c) -> max3(a, b, c)
     // min(min(a, b), c) -> min3(a, b, c)
     if (Op0.getOpcode() == Opc && Op0.hasOneUse()) {
       SDLoc DL(N);
       return DAG.getNode(minMaxOpcToMin3Max3Opc(Opc),
                          DL,
                          N->getValueType(0),
                          Op0.getOperand(0),
if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
     if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, false))
       return Med3;
   }
 
   // fminnum(fmaxnum(x, K0), K1), K0 < K1 && !is_snan(x) -> fmed3(x, K0, K1)
   if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
        (Opc == AMDGPUISD::FMIN_LEGACY &&
         Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
-      (N->getValueType(0) == MVT::f32 ||
-       (N->getValueType(0) == MVT::f16 && Subtarget->has16BitInsts())) &&
+      (VT == MVT::f32 || VT == MVT::f64 ||
+       (VT == MVT::f16 && Subtarget->has16BitInsts())) &&
       Op0.hasOneUse()) {
     if (SDValue Res = performFPMed3ImmCombine(DAG, SDLoc(N), Op0, Op1))
       return Res;
   }
 
   return SDValue();
 }
 
SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::FMINNUM:
   case ISD::SMAX:
   case ISD::SMIN:
   case ISD::UMAX:
   case ISD::UMIN:
   case AMDGPUISD::FMIN_LEGACY:
   case AMDGPUISD::FMAX_LEGACY: {
     if (DCI.getDAGCombineLevel() >= AfterLegalizeDAG &&
-        N->getValueType(0) != MVT::f64 &&
         getTargetMachine().getOptLevel() > CodeGenOpt::None)
       return performMinMaxCombine(N, DCI);
     break;
   }
   case ISD::LOAD:
   case ISD::STORE:
   case ISD::ATOMIC_LOAD:
   case ISD::ATOMIC_STORE:

lib/Target/AMDGPU/SIInstructions.td

 // inputs. The output result is not flushed.
 class ClampPat<Instruction inst, ValueType vt> : Pat <
   (vt (AMDGPUclamp
         (VOP3Mods0Clamp vt:$src0, i32:$src0_modifiers, i32:$omod))),
   (inst i32:$src0_modifiers, vt:$src0,
         i32:$src0_modifiers, vt:$src0, DSTCLAMP.ENABLE, $omod)
 >;
 
-// TODO: Does f64 support clamp?
 def : ClampPat<V_MAX_F32_e64, f32>;
+def : ClampPat<V_MAX_F64, f64>;
 def : ClampPat<V_MAX_F16_e64, f16>;
 
 /********** ================================ **********/
 /********** Floating point absolute/negative **********/
 /********** ================================ **********/
 
 // Prevent expanding both fneg and fabs.
 

test/CodeGen/AMDGPU/clamp-modifier.ll

define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %aptr) #0 {
   %a = load <2 x float>, <2 x float> addrspace(1)* %gep0
   %add = fadd <2 x float> %a, <float 1.0, float 1.0>
   %max = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %add, <2 x float> zeroinitializer)
   %clamp = call <2 x float> @llvm.minnum.v2f32(<2 x float> %max, <2 x float> <float 1.0, float 1.0>)
   store <2 x float> %clamp, <2 x float> addrspace(1)* %out.gep
   ret void
 }
 
+; GCN-LABEL: {{^}}v_clamp_add_src_f64:
+; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
+; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], 1.0 clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr double, double addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr double, double addrspace(1)* %out, i32 %tid
+  %a = load double, double addrspace(1)* %gep0
+  %add = fadd double %a, 1.0
+  %max = call double @llvm.maxnum.f64(double %add, double 0.0)
+  %clamp = call double @llvm.minnum.f64(double %max, double 1.0)
+  store double %clamp, double addrspace(1)* %out.gep
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare float @llvm.fabs.f32(float) #1
 declare float @llvm.floor.f32(float) #1
 declare float @llvm.minnum.f32(float, float) #1
 declare float @llvm.maxnum.f32(float, float) #1
 declare float @llvm.amdgcn.fmed3.f32(float, float, float) #1
 declare double @llvm.fabs.f64(double) #1
 declare double @llvm.minnum.f64(double, double) #1

test/CodeGen/AMDGPU/clamp.ll

define amdgpu_kernel void @v_clamp_negabs_f16(half addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
 
   store half %med, half addrspace(1)* %out.gep
   ret void
 }
 
 ; FIXME: Do f64 instructions support clamp?
 ; GCN-LABEL: {{^}}v_clamp_f64:
 ; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
-; GCN: v_max_f64
-; GCN: v_min_f64
+; GCN: v_max_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr double, double addrspace(1)* %aptr, i32 %tid
   %out.gep = getelementptr double, double addrspace(1)* %out, i32 %tid
   %a = load double, double addrspace(1)* %gep0
   %max = call double @llvm.maxnum.f64(double %a, double 0.0)
   %med = call double @llvm.minnum.f64(double %max, double 1.0)
 
   store double %med, double addrspace(1)* %out.gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_clamp_neg_f64:
 ; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
-; GCN: v_max_f64
-; GCN: v_min_f64
+; GCN: v_max_f64 v{{\[[0-9]+:[0-9]+\]}}, -[[A]], -[[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_neg_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr double, double addrspace(1)* %aptr, i32 %tid
   %out.gep = getelementptr double, double addrspace(1)* %out, i32 %tid
   %a = load double, double addrspace(1)* %gep0
   %fneg.a = fsub double -0.0, %a
   %max = call double @llvm.maxnum.f64(double %fneg.a, double 0.0)
   %med = call double @llvm.minnum.f64(double %max, double 1.0)
 
   store double %med, double addrspace(1)* %out.gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_clamp_negabs_f64:
 ; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
-; GCN: v_max_f64
-; GCN: v_min_f64
+; GCN: v_max_f64 v{{\[[0-9]+:[0-9]+\]}}, -|[[A]]|, -|[[A]]| clamp{{$}}
 define amdgpu_kernel void @v_clamp_negabs_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr double, double addrspace(1)* %aptr, i32 %tid
   %out.gep = getelementptr double, double addrspace(1)* %out, i32 %tid
   %a = load double, double addrspace(1)* %gep0
   %fabs.a = call double @llvm.fabs.f64(double %a)
   %fneg.fabs.a = fsub double -0.0, %fabs.a