AMDGPU: Fold FP clamp as modifier bit

[arsenm]

Summary

The manual is unclear on the details of this. It's not
clear to me if denormals are not allowed with clamp,
or if that is only omod. Not allowing denorms for
fp16 or fp64 isn't useful so I also question if that
is really a restriction. Same with whether this is valid
without IEEE mode enabled.

[mareko]

I only know that exceptions won't occur with the clamp modifier. No idea about denormals.

Also, shouldn't this handle MIN as well?

[arsenm]

In D30134#681271, @mareko wrote:

I only know that exceptions won't occur with the clamp modifier. No idea about denormals.

Also, shouldn't this handle MIN as well?

There's no practical reason to handle min. The higher level operation minnum(x, x) is folded to x in the IR, so this should only be appearing when we emit this pattern for the clamp operation, where max was arbitrarily chosen.

[mareko]

In D30134#682357, @arsenm wrote:

In D30134#681271, @mareko wrote:

I only know that exceptions won't occur with the clamp modifier. No idea about denormals.

Also, shouldn't this handle MIN as well?

There's no practical reason to handle min. The higher level operation minnum(x, x) is folded to x in the IR, so this should only be appearing when we emit this pattern for the clamp operation, where max was arbitrarily chosen.

I don't understand. FPClamp(x) = min(max(x, 0), 1). I don't see min handled here, that's why I asked.

[arsenm]

In D30134#682362, @mareko wrote:

In D30134#682357, @arsenm wrote:

In D30134#681271, @mareko wrote:

I only know that exceptions won't occur with the clamp modifier. No idea about denormals.

Also, shouldn't this handle MIN as well?

There's no practical reason to handle min. The higher level operation minnum(x, x) is folded to x in the IR, so this should only be appearing when we emit this pattern for the clamp operation, where max was arbitrarily chosen.

I don't understand. FPClamp(x) = min(max(x, 0), 1). I don't see min handled here, that's why I asked.

This isn't directly matching the clamp pattern. In the DAG we match that to AMDGPUISD::CLAMP. We emit that as the max(x, x) clamp. We match that here

[arsenm]

Allow with denormals enabled

[mareko]

LGTM.

[arsenm]

r295905

[foad]

test/CodeGen/AMDGPU/clamp.ll

108–109

Does this FIXME still make sense? Not sure what it was trying to say in the first place.

128

Ditto.

[arsenm]

test/CodeGen/AMDGPU/clamp.ll

108–109

The source modifier is used here, so I don't think so

lib/Target/AMDGPU/SIDefines.h

// TODO: Should this be spilt into VOP3 a and b?
   LGKM_CNT = UINT64_C(1) << 34,
 
   WQM = UINT64_C(1) << 35,
   DisableWQM = UINT64_C(1) << 36,
   Gather4 = UINT64_C(1) << 37,
   SOPK_ZEXT = UINT64_C(1) << 38,
   SCALAR_STORE = UINT64_C(1) << 39,
   FIXED_SIZE = UINT64_C(1) << 40,
-  VOPAsmPrefer32Bit = UINT64_C(1) << 41
-
+  VOPAsmPrefer32Bit = UINT64_C(1) << 41,
+  HasFPClamp = UINT64_C(1) << 42
 };
 
 // v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
 // The result is true if any of these tests are true.
 enum ClassFlags {
   S_NAN = 1 << 0,        // Signaling NaN
   Q_NAN = 1 << 1,        // Quiet NaN
   N_INFINITY = 1 << 2,   // Negative infinity

lib/Target/AMDGPU/SIFoldOperands.cpp

 };
 
 class SIFoldOperands : public MachineFunctionPass {
 public:
   static char ID;
   MachineRegisterInfo *MRI;
   const SIInstrInfo *TII;
   const SIRegisterInfo *TRI;
+  const SISubtarget *ST;
 
   void foldOperand(MachineOperand &OpToFold,
                    MachineInstr *UseMI,
                    unsigned UseOpIdx,
                    SmallVectorImpl<FoldCandidate> &FoldList,
                    SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
 
   void foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;
 
+  const MachineOperand *isClamp(const SIInstrInfo &TII,
+                                const MachineInstr &MI) const;
+  bool tryFoldClamp(MachineInstr &MI);
+
 public:
   SIFoldOperands() : MachineFunctionPass(ID) {
     initializeSIFoldOperandsPass(*PassRegistry::getPassRegistry());
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   StringRef getPassName() const override { return "SI Fold Operands"; }
if (updateOperand(Fold, *TRI)) {
         MRI->clearKillFlags(Fold.OpToFold->getReg());
       }
       DEBUG(dbgs() << "Folded source from " << MI << " into OpNo " <<
             static_cast<int>(Fold.UseOpNo) << " of " << *Fold.UseMI << '\n');
     }
   }
 }
 
+const MachineOperand *SIFoldOperands::isClamp(const SIInstrInfo &TII,
+                                              const MachineInstr &MI) const {
+  unsigned Op = MI.getOpcode();
+  switch (Op) {
+  case AMDGPU::V_MAX_F32_e64:
+  case AMDGPU::V_MAX_F16_e64: {
+    // XXX - Do denormals work with clamp? At least for f16?
+    if ((ST->hasFP32Denormals() && Op == AMDGPU::V_MAX_F32_e64) ||
+        (ST->hasFP16Denormals() && Op == AMDGPU::V_MAX_F16_e64))
+      return nullptr;
+
+    if (!TII.getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
+      return nullptr;
+
+    // Make sure sources are identical.
+    const MachineOperand *Src0 = TII.getNamedOperand(MI, AMDGPU::OpName::src0);
+    const MachineOperand *Src1 = TII.getNamedOperand(MI, AMDGPU::OpName::src1);
+    if (!Src0->isReg() || Src0->getSubReg() != Src1->getSubReg() ||
+        Src0->getSubReg() != AMDGPU::NoSubRegister)
+      return nullptr;
+
+    // Can't fold up if we have source modifiers.
+    if (TII.hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
+        TII.hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))
+      return nullptr;
+    return Src0;
+  }
+  default:
+    return nullptr;
+  }
+}
+
+// We obviously have multiple uses in a clamp since the register is used twice
+// in the same instruction.
+static bool hasOneNonDBGUseInst(const MachineRegisterInfo &MRI, unsigned Reg) {
+  int Count = 0;
+  for (auto I = MRI.use_instr_nodbg_begin(Reg), E = MRI.use_instr_nodbg_end();
+       I != E; ++I) {
+    if (++Count > 1)
+      return false;
+  }
+
+  return true;
+}
+
+// FIXME: Does this need to check IEEE bit on function?
+bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) {
+  const MachineOperand *ClampSrc = isClamp(*TII, MI);
+  if (!ClampSrc || !hasOneNonDBGUseInst(*MRI, ClampSrc->getReg()))
+    return false;
+
+  MachineInstr *Def = MRI->getUniqueVRegDef(ClampSrc->getReg());
+  if (!TII->hasFPClamp(*Def))
+    return false;
+  MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp);
+  if (!DefClamp)
+    return false;
+
+  DefClamp->setImm(1);
+  MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
+  MI.eraseFromParent();
+  return true;
+}
+
 bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(*MF.getFunction()))
     return false;
 
-  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
-
   MRI = &MF.getRegInfo();
-  TII = ST.getInstrInfo();
+  ST = &MF.getSubtarget<SISubtarget>();
+  TII = ST->getInstrInfo();
   TRI = &TII->getRegisterInfo();
 
   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
        BI != BE; ++BI) {
 
     MachineBasicBlock &MBB = *BI;
     MachineBasicBlock::iterator I, Next;
     for (I = MBB.begin(); I != MBB.end(); I = Next) {
       Next = std::next(I);
       MachineInstr &MI = *I;
 
-      if (!isSafeToFold(MI))
+      if (!isSafeToFold(MI)) {
+        // TODO: Try omod also.
+        tryFoldClamp(MI);
         continue;
+      }
 
       MachineOperand &OpToFold = MI.getOperand(1);
       bool FoldingImm = OpToFold.isImm() || OpToFold.isFI();
 
       // FIXME: We could also be folding things like TargetIndexes.
       if (!FoldingImm && !OpToFold.isReg())
         continue;
 

lib/Target/AMDGPU/SIInstrFormats.td

class InstSI <dag outs, dag ins, string asm = "",
   // Whether the operands can be ignored when computing the
   // instruction size.
   field bit FixedSize = 0;
 
   // This bit tells the assembler to use the 32-bit encoding in case it
   // is unable to infer the encoding from the operands.
   field bit VOPAsmPrefer32Bit = 0;
 
+  // This bit indicates that this has a floating point result type, so
+  // the clamp modifier has floating point semantics.
+  field bit FPClamp = 0;
+
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = SALU;
   let TSFlags{1} = VALU;
 
   let TSFlags{2} = SOP1;
   let TSFlags{3} = SOP2;
   let TSFlags{4} = SOPC;
   let TSFlags{5} = SOPK;
class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{35} = WQM;
   let TSFlags{36} = DisableWQM;
   let TSFlags{37} = Gather4;
 
   let TSFlags{38} = SOPKZext;
   let TSFlags{39} = ScalarStore;
   let TSFlags{40} = FixedSize;
   let TSFlags{41} = VOPAsmPrefer32Bit;
+  let TSFlags{42} = FPClamp;
 
   let SchedRW = [Write32Bit];
 
   field bits<1> DisableSIDecoder = 0;
   field bits<1> DisableVIDecoder = 0;
   field bits<1> DisableDecoder = 0;
 
   let isAsmParserOnly = !if(!eq(DisableDecoder{0}, {0}), 0, 1);

lib/Target/AMDGPU/SIInstrInfo.h

public:
   static bool isFixedSize(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::FIXED_SIZE;
   }
 
   bool isFixedSize(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::FIXED_SIZE;
   }
 
+  static bool hasFPClamp(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::HasFPClamp;
+  }
+
+  bool hasFPClamp(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::HasFPClamp;
+  }
+
   bool isVGPRCopy(const MachineInstr &MI) const {
     assert(MI.isCopy());
     unsigned Dest = MI.getOperand(0).getReg();
     const MachineFunction &MF = *MI.getParent()->getParent();
     const MachineRegisterInfo &MRI = MF.getRegInfo();
     return !RI.isSGPRReg(MRI, Dest);
   }
 

lib/Target/AMDGPU/SIInstrInfo.td

class VOPProfile <list<ValueType> _ArgVT> {
 
   field bit HasSrc0Mods = HasModifiers;
   field bit HasSrc1Mods = !if(HasModifiers, BitOr<HasSrc1FloatMods, HasSrc1IntMods>.ret, 0);
   field bit HasSrc2Mods = !if(HasModifiers, BitOr<HasSrc2FloatMods, HasSrc2IntMods>.ret, 0);
 
   field bit HasOMod = HasModifiers;
   field bit HasClamp = HasModifiers;
   field bit HasSDWAClamp = HasSrc0;
+  field bit HasFPClamp = BitAnd<isFloatType<DstVT>.ret, HasClamp>.ret;
 
   field bit HasExt = getHasExt<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret;
 
   field dag Outs = !if(HasDst,(outs DstRC:$vdst),(outs));
 
   // VOP3b instructions are a special case with a second explicit
   // output. This is manually overridden for them.
   field dag Outs32 = Outs;

lib/Target/AMDGPU/VOPInstructions.td

class VOP3_Pseudo <string opName, VOPProfile P, list<dag> pattern=[], bit VOP3Only = 0> :
   // Using complex patterns gives VOP3 patterns a very high complexity rating,
   // but standalone patterns are almost always preferred, so we need to adjust the
   // priority lower.  The goal is to use a high number to reduce complexity to
   // zero (or less than zero).
   let AddedComplexity = -1000;
 
   let VOP3 = 1;
   let VALU = 1;
+  let FPClamp = P.HasFPClamp;
   let Uses = [EXEC];
 
   let AsmVariantName = AMDGPUAsmVariants.VOP3;
   let AsmMatchConverter =
     !if(!eq(VOP3Only,1),
         "cvtVOP3",
         !if(!eq(P.HasModifiers, 1), "cvtVOP3_2_mod", ""));
 

test/CodeGen/AMDGPU/clamp.ll

define amdgpu_kernel void @v_clamp_multi_use_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   store volatile float %max, float addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_clamp_f16:
 ; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
 ; VI: v_max_f16_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 
-; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
-; SI: v_max_f32_e64 v{{[0-9]+}}, [[CVT]], [[CVT]] clamp{{$}}
-; SI: v_cvt_f16_f32_e32
+; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], [[A]] clamp{{$}}
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, [[CVT]]
 define amdgpu_kernel void @v_clamp_f16(half addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid
   %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid
   %a = load half, half addrspace(1)* %gep0
   %max = call half @llvm.maxnum.f16(half %a, half 0.0)
   %med = call half @llvm.minnum.f16(half %max, half 1.0)
 
   store half %med, half addrspace(1)* %out.gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_clamp_neg_f16:
 ; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
 ; VI: v_max_f16_e64 v{{[0-9]+}}, -[[A]], -[[A]] clamp{{$}}
 
 ; FIXME: Better to fold neg into max
-; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -[[A]]
-; SI: v_max_f32_e64 v{{[0-9]+}}, [[CVT]], [[CVT]] clamp{{$}}
-; SI: v_cvt_f16_f32
+; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -[[A]] clamp{{$}}
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, [[CVT]]
 define amdgpu_kernel void @v_clamp_neg_f16(half addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid
   %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid
   %a = load half, half addrspace(1)* %gep0
   %fneg.a = fsub half -0.0, %a
   %max = call half @llvm.maxnum.f16(half %fneg.a, half 0.0)
   %med = call half @llvm.minnum.f16(half %max, half 1.0)
 
   store half %med, half addrspace(1)* %out.gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_clamp_negabs_f16:
 ; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
 ; VI: v_max_f16_e64 v{{[0-9]+}}, -|[[A]]|, -|[[A]]| clamp{{$}}
 
 ; FIXME: Better to fold neg/abs into max
 
-; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -|[[A]]|
-; SI: v_max_f32_e64 v{{[0-9]+}}, [[CVT]], [[CVT]] clamp{{$}}
-; SI: v_cvt_f16_f32_e32
+; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -|[[A]]| clamp{{$}}
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, [[CVT]]
 define amdgpu_kernel void @v_clamp_negabs_f16(half addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid
   %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid
   %a = load half, half addrspace(1)* %gep0
   %fabs.a = call half @llvm.fabs.f16(half %a)
   %fneg.fabs.a = fsub half -0.0, %fabs.a