[AMDGPU] Tune inlining parameters for AMDGPU target

[dfukalov]

Summary

Since the target has no significant advantage of vectorization,
vector instructions bous threshold bonus should be optional.

amdgpu-inline-arg-alloca-cost parameter default value and the target
InliningThresholdMultiplier value tuned then respectively.

[arsenm]

llvm/include/llvm/Analysis/TargetTransformInfo.h

276

I think this need a name indicating it's an inliner control. getInlinerVectorBonusPercent?

llvm/lib/Analysis/InlineCost.cpp

883

How does it decide what "vector dense" means? We already report costs that approximately say scalarize everything, and scalarization is free

llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll

28–41

Why this test change? I would expect a separate version without the control flow?

[rampitec]

Agree with Matt on the callback name change. Otherwise LGTM.

[dfukalov]

llvm/lib/Analysis/InlineCost.cpp

883

They estimate this "dense" by a percent of LLVM IR instructions with vector arguments. So if a function contains more than 50% of vector instructions this bonus added to threshold. For 10%-50% vector instructions cases they add half of the bonus.
I guess this logic of bonuses is based on x86 extensions like MMX and others.

llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll

28–41

Without the modification test @test_inliner_multi_pvt_ptr_cutoff starts to fail since I decreased the threshold multiplier and cost of the function started to be slightly higher.
The test is not about cotrol flow, we should check amdgpu-inline-arg-alloca-cutoff value: foo_private_ptr2 should be inlined in test_inliner_multi_pvt_ptr and shouldn't be inlined in test_inliner_multi_pvt_ptr_cutoff

[dfukalov]

Diff updated as requested

[rampitec]

LGTM

[eraman]

llvm/lib/Analysis/InlineCost.cpp

883

The comment block explaining vector bonuses is still relevant after this change. Instead of removing it, you should modify it to say the bonus percentage is target dependent.

[dfukalov]

llvm/lib/Analysis/InlineCost.cpp

883

the comment was not removed but moved to TargetTransformInfo.h where to new function. And note about target was added also.

llvm/include/llvm/Analysis/TargetTransformInfo.h

public:
   /// \returns A value by which our inlining threshold should be multiplied.
   /// This is primarily used to bump up the inlining threshold wholesale on
   /// targets where calls are unusually expensive.
   ///
   /// TODO: This is a rather blunt instrument.  Perhaps altering the costs of
   /// individual classes of instructions would be better.
   unsigned getInliningThresholdMultiplier() const;
 
+  /// \returns Vector bonus in percent.
+  ///
+  /// Vector bonuses: We want to more aggressively inline vector-dense kernels
+  /// and apply this bonus based on the percentage of vector instructions. A
+  /// bonus is applied if the vector instructions exceed 50% and half that amount
+  /// is applied if it exceeds 10%. Note that these bonuses are some what
+  /// arbitrary and evolved over time by accident as much as because they are
+  /// principled bonuses.
+  /// FIXME: It would be nice to base the bonus values on something more
+  /// scientific. A target may has no bonus on vector instructions.
+  int getVectorBonusPercent() const;
+
   /// Estimate the cost of an intrinsic when lowered.
   ///
   /// Mirrors the \c getCallCost method but uses an intrinsic identifier.
   int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                        ArrayRef<Type *> ParamTys,
                        const User *U = nullptr) const;
 
   /// Estimate the cost of an intrinsic when lowered.
public:
   virtual int getGEPCost(Type *PointeeType, const Value *Ptr,
                          ArrayRef<const Value *> Operands) = 0;
   virtual int getExtCost(const Instruction *I, const Value *Src) = 0;
   virtual int getCallCost(FunctionType *FTy, int NumArgs, const User *U) = 0;
   virtual int getCallCost(const Function *F, int NumArgs, const User *U) = 0;
   virtual int getCallCost(const Function *F,
                           ArrayRef<const Value *> Arguments, const User *U) = 0;
   virtual unsigned getInliningThresholdMultiplier() = 0;
+  virtual int getVectorBonusPercent() = 0;
   virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                                ArrayRef<Type *> ParamTys, const User *U) = 0;
   virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                                ArrayRef<const Value *> Arguments,
                                const User *U) = 0;
   virtual int getMemcpyCost(const Instruction *I) = 0;
   virtual unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
                                                     unsigned &JTSize) = 0;
public:
   }
   int getCallCost(const Function *F,
                   ArrayRef<const Value *> Arguments, const User *U) override {
     return Impl.getCallCost(F, Arguments, U);
   }
   unsigned getInliningThresholdMultiplier() override {
     return Impl.getInliningThresholdMultiplier();
   }
+  int getVectorBonusPercent() override {
+    return Impl.getVectorBonusPercent();
+  }
   int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                        ArrayRef<Type *> ParamTys, const User *U = nullptr) override {
     return Impl.getIntrinsicCost(IID, RetTy, ParamTys, U);
   }
   int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                        ArrayRef<const Value *> Arguments,
                        const User *U = nullptr) override {
     return Impl.getIntrinsicCost(IID, RetTy, Arguments, U);

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

if (NumArgs < 0)
       // function.
       NumArgs = FTy->getNumParams();
 
     return TTI::TCC_Basic * (NumArgs + 1);
   }
 
   unsigned getInliningThresholdMultiplier() { return 1; }
 
+  int getVectorBonusPercent() { return 150; }
+
   unsigned getMemcpyCost(const Instruction *I) {
     return TTI::TCC_Expensive;
   }
 
   bool hasBranchDivergence() { return false; }
 
   bool isSourceOfDivergence(const Value *V) { return false; }
 

llvm/include/llvm/CodeGen/BasicTTIImpl.h

case Instruction::AddrSpaceCast:
       return TargetTransformInfo::TCC_Basic;
     }
 
     return BaseT::getOperationCost(Opcode, Ty, OpTy);
   }
 
   unsigned getInliningThresholdMultiplier() { return 1; }
 
+  int getVectorBonusPercent() { return 150; }
+
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP) {
     // This unrolling functionality is target independent, but to provide some
     // motivation for its intended use, for x86:
 
     // According to the Intel 64 and IA-32 Architectures Optimization Reference
     // Manual, Intel Core models and later have a loop stream detector (and
     // associated uop queue) that can benefit from partial unrolling.

llvm/lib/Analysis/InlineCost.cpp

void CallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) {
   };
 
   // Various bonus percentages. These are multiplied by Threshold to get the
   // bonus values.
   // SingleBBBonus: This bonus is applied if the callee has a single reachable
   // basic block at the given callsite context. This is speculatively applied
   // and withdrawn if more than one basic block is seen.
   //
-  // Vector bonuses: We want to more aggressively inline vector-dense kernels
-  // and apply this bonus based on the percentage of vector instructions. A
-  // bonus is applied if the vector instructions exceed 50% and half that amount
-  // is applied if it exceeds 10%. Note that these bonuses are some what
-  // arbitrary and evolved over time by accident as much as because they are
-  // principled bonuses.
-  // FIXME: It would be nice to base the bonus values on something more
-  // scientific.
-  //
   // LstCallToStaticBonus: This large bonus is applied to ensure the inlining
   // of the last call to a static function as inlining such functions is
   // guaranteed to reduce code size.
   //
   // These bonus percentages may be set to 0 based on properties of the caller
   // and the callsite.
   int SingleBBBonusPercent = 50;
-  int VectorBonusPercent = 150;
+  int VectorBonusPercent = TTI.getVectorBonusPercent();
   int LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus;
 
   // Lambda to set all the above bonus and bonus percentages to 0.
   auto DisallowAllBonuses = [&]() {
     SingleBBBonusPercent = 0;
     VectorBonusPercent = 0;
     LastCallToStaticBonus = 0;
   };

llvm/lib/Analysis/TargetTransformInfo.cpp

int TargetTransformInfo::getCallCost(const Function *F,
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
 
 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
   return TTIImpl->getInliningThresholdMultiplier();
 }
 
+int TargetTransformInfo::getVectorBonusPercent() const {
+  return TTIImpl->getVectorBonusPercent();
+}
+
 int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr,
                                     ArrayRef<const Value *> Operands) const {
   return TTIImpl->getGEPCost(PointeeType, Ptr, Operands);
 }
 
 int TargetTransformInfo::getExtCost(const Instruction *I,
                                     const Value *Src) const {
   return TTIImpl->getExtCost(I, Src);

llvm/lib/Target/AMDGPU/AMDGPUInline.cpp

 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/IPO/Inliner.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "inline"
 
 static cl::opt<int>
-ArgAllocaCost("amdgpu-inline-arg-alloca-cost", cl::Hidden, cl::init(2200),
+ArgAllocaCost("amdgpu-inline-arg-alloca-cost", cl::Hidden, cl::init(1500),
               cl::desc("Cost of alloca argument"));
 
 // If the amount of scratch memory to eliminate exceeds our ability to allocate
 // it into registers we gain nothing by aggressively inlining functions for that
 // heuristic.
 static cl::opt<unsigned>
 ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden, cl::init(256),
                 cl::desc("Maximum alloca size to use for inline cost"));

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

public:
   unsigned getVectorSplitCost() { return 0; }
 
   unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
                           Type *SubTp);
 
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
-  unsigned getInliningThresholdMultiplier() { return 9; }
+  unsigned getInliningThresholdMultiplier() { return 7; }
+
+  int getVectorBonusPercent() { return 0; }
 
   int getArithmeticReductionCost(unsigned Opcode,
                                  Type *Ty,
                                  bool IsPairwise);
   int getMinMaxReductionCost(Type *Ty, Type *CondTy,
                              bool IsPairwiseForm,
                              bool IsUnsigned);
 };

llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll

 ; RUN: opt -mtriple=amdgcn--amdhsa -data-layout=A5 -O3 -S -inline-threshold=1 < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-INL1 %s
-; RUN: opt -mtriple=amdgcn--amdhsa -data-layout=A5 -O3 -S < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-INLDEF %s
+; RUN: opt -mtriple=amdgcn--amdhsa -data-layout=A5 -O3 -S -amdgpu-inline-arg-alloca-cutoff=260 < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-INLDEF %s
 
 define coldcc float @foo(float %x, float %y) {
 entry:
   %cmp = fcmp ogt float %x, 0.000000e+00
   %div = fdiv float %y, %x
   %mul = fmul float %x, %y
   %cond = select i1 %cmp, float %div, float %mul
   ret float %cond
if.then:                                          ; preds = %entry
   %div = fdiv float 1.000000e+00, %tmp1
   store float %div, float addrspace(5)* %p, align 4
   br label %if.end
 
 if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
 define coldcc void @foo_private_ptr2(float addrspace(5)* nocapture %p1, float addrspace(5)* nocapture %p2) {
 entry:
   %tmp1 = load float, float addrspace(5)* %p1, align 4
-  %cmp = fcmp ogt float %tmp1, 1.000000e+00
-  br i1 %cmp, label %if.then, label %if.end
-
-if.then:                                          ; preds = %entry
   %div = fdiv float 2.000000e+00, %tmp1
   store float %div, float addrspace(5)* %p2, align 4
-  br label %if.end
-
-if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
 define coldcc float @sin_wrapper(float %x) {
 bb:
   %call = tail call float @_Z3sinf(float %x)
   ret float %call
 }
 
 define void @foo_noinline(float addrspace(5)* nocapture %p) #0 {

llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-vecbonus.ll

+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -amdgpu-inline --inline-threshold=1 < %s | FileCheck %s
+
+define hidden <16 x i32> @div_vecbonus(<16 x i32> %x, <16 x i32> %y) {
+entry:
+  %div.1 = udiv <16 x i32> %x, %y
+  %div.2 = udiv <16 x i32> %div.1, %y
+  %div.3 = udiv <16 x i32> %div.2, %y
+  %div.4 = udiv <16 x i32> %div.3, %y
+  %div.5 = udiv <16 x i32> %div.4, %y
+  %div.6 = udiv <16 x i32> %div.5, %y
+  %div.7 = udiv <16 x i32> %div.6, %y
+  %div.8 = udiv <16 x i32> %div.7, %y
+  %div.9 = udiv <16 x i32> %div.8, %y
+  %div.10 = udiv <16 x i32> %div.9, %y
+  %div.11 = udiv <16 x i32> %div.10, %y
+  %div.12 = udiv <16 x i32> %div.11, %y
+  ret <16 x i32> %div.12
+}
+
+; CHECK-LABEL: define amdgpu_kernel void @caller_vecbonus
+; CHECK-NOT: udiv
+; CHECK: tail call <16 x i32> @div_vecbonus
+; CHECK: ret void
+define amdgpu_kernel void @caller_vecbonus(<16 x i32> addrspace(1)* nocapture %x, <16 x i32> addrspace(1)* nocapture readonly %y) {
+entry:
+  %tmp = load <16 x i32>, <16 x i32> addrspace(1)* %x
+  %tmp1 = load <16 x i32>, <16 x i32> addrspace(1)* %y
+  %div.i = tail call <16 x i32> @div_vecbonus(<16 x i32> %tmp, <16 x i32> %tmp1)
+  store <16 x i32> %div.i, <16 x i32> addrspace(1)* %x
+  ret void
+}
+
+