[NFC][TargetTransformInfo] Make getInliningThreholdMultiplier and getInlinerVectorBonusPercent const

[jmmartinez]

Summary

[InlineCost][TargetTransformInfo][AMDGPU] Consider cost of alloca instructions in the caller (1/2)

On AMDGPU, alloca instructions have penalty that can
be avoided when SROA is applied after inlining.

This patch introduces the default implementation of
TargetTransformInfo::getCallerAllocaCost.

[InlineCost][TargetTransformInfo][AMDGPU] Consider cost of alloca instructions in the caller (2/2)

Before this patch, the compiler gave a bump to the inline-threshold
when the total size of the allocas passed as arguments to the
callee was below 256 bytes.
This heuristic ignores that some of these allocas could have be removed
by SROA if inlining was applied.

Ideally, this bonus would be attributed to the threshold once the
size of all the allocas that could not be handled by SROA is known:
at the end of the InlineCost analysis.
However, we may never reach this point if the inline-cost analysis exits
early when the inline cost goes over the threshold mid-analysis.

This patch proposes:

• Attribute the bonus in the inline-threshold when allocas are passed as arguments (regardless of their total size).
• Assigns a cost to each alloca proportional to its size, such that the cost of all the allocas cancels the bonus.

Potential problems:

• This patch assumes that removing alloca instructions with SROA is always profitable. This may not be the case if the total size of the allocas is still too big to be promoted to registers/LDS.
• Redundant calls to getTotalAllocaSize
• Awkwardly, the threshold attributed contributes to the single-bb and vector bonus.

[jmmartinez]

Bad manipulation.

llvm/include/llvm/Analysis/TargetTransformInfo.h

 #include <utility>
 
 namespace llvm {
 
 namespace Intrinsic {
 typedef unsigned ID;
 }
 
+class AllocaInst;
 class AssumptionCache;
 class BlockFrequencyInfo;
 class DominatorTree;
 class BranchInst;
 class CallBase;
 class Function;
 class GlobalValue;
 class InstCombiner;
public:
   ///
   /// TODO: This is a rather blunt instrument.  Perhaps altering the costs of
   /// individual classes of instructions would be better.
   unsigned getInliningThresholdMultiplier() const;
 
   /// \returns A value to be added to the inlining threshold.
   unsigned adjustInliningThreshold(const CallBase *CB) const;
 
+  /// \returns The cost of having an Alloca in the caller if not inlined, to be
+  /// added to the threshold
+  unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const;
+
   /// \returns Vector bonus in percent.
   ///
   /// Vector bonuses: We want to more aggressively inline vector-dense kernels
   /// and apply this bonus based on the percentage of vector instructions. A
   /// bonus is applied if the vector instructions exceed 50% and half that
   /// amount is applied if it exceeds 10%. Note that these bonuses are some what
   /// arbitrary and evolved over time by accident as much as because they are
   /// principled bonuses.
public:
   virtual const DataLayout &getDataLayout() const = 0;
   virtual InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
                                      ArrayRef<const Value *> Operands,
                                      TTI::TargetCostKind CostKind) = 0;
   virtual InstructionCost
   getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
                        const TTI::PointersChainInfo &Info,
                        TTI::TargetCostKind CostKind) = 0;
-  virtual unsigned getInliningThresholdMultiplier() = 0;
+  virtual unsigned getInliningThresholdMultiplier() const = 0;
   virtual unsigned adjustInliningThreshold(const CallBase *CB) = 0;
-  virtual int getInlinerVectorBonusPercent() = 0;
+  virtual int getInlinerVectorBonusPercent() const = 0;
+  virtual unsigned getCallerAllocaCost(const CallBase *CB,
+                                       const AllocaInst *AI) const = 0;
   virtual InstructionCost getMemcpyCost(const Instruction *I) = 0;
   virtual unsigned
   getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JTSize,
                                    ProfileSummaryInfo *PSI,
                                    BlockFrequencyInfo *BFI) = 0;
   virtual InstructionCost getInstructionCost(const User *U,
                                              ArrayRef<const Value *> Operands,
                                              TargetCostKind CostKind) = 0;
getGEPCost(Type *PointeeType, const Value *Ptr,
     return Impl.getGEPCost(PointeeType, Ptr, Operands, CostKind);
   }
   InstructionCost getPointersChainCost(ArrayRef<const Value *> Ptrs,
                                        const Value *Base,
                                        const PointersChainInfo &Info,
                                        TargetCostKind CostKind) override {
     return Impl.getPointersChainCost(Ptrs, Base, Info, CostKind);
   }
-  unsigned getInliningThresholdMultiplier() override {
+  unsigned getInliningThresholdMultiplier() const override {
     return Impl.getInliningThresholdMultiplier();
   }
   unsigned adjustInliningThreshold(const CallBase *CB) override {
     return Impl.adjustInliningThreshold(CB);
-  }
-  int getInlinerVectorBonusPercent() override {
+  } 
+  int getInlinerVectorBonusPercent() const override {
     return Impl.getInlinerVectorBonusPercent();
   }
+  unsigned getCallerAllocaCost(const CallBase *CB,
+                               const AllocaInst *AI) const override {
+    return Impl.getCallerAllocaCost(CB, AI);
+  }
   InstructionCost getMemcpyCost(const Instruction *I) override {
     return Impl.getMemcpyCost(I);
   }
   InstructionCost getInstructionCost(const User *U,
                                      ArrayRef<const Value *> Operands,
                                      TargetCostKind CostKind) override {
     return Impl.getInstructionCost(U, Operands, CostKind);
   }

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
     (void)PSI;
     (void)BFI;
     JTSize = 0;
     return SI.getNumCases();
   }
 
   unsigned getInliningThresholdMultiplier() const { return 1; }
   unsigned adjustInliningThreshold(const CallBase *CB) const { return 0; }
+  unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
+    return 0;
+  };
 
   int getInlinerVectorBonusPercent() const { return 150; }
 
   InstructionCost getMemcpyCost(const Instruction *I) const {
     return TTI::TCC_Expensive;
   }
 
   // Although this default value is arbitrary, it is not random. It is assumed

llvm/include/llvm/CodeGen/BasicTTIImpl.h

InstructionCost getFPOpCost(Type *Ty) {
     // general.
     const TargetLoweringBase *TLI = getTLI();
     EVT VT = TLI->getValueType(DL, Ty);
     if (TLI->isOperationLegalOrCustomOrPromote(ISD::FADD, VT))
       return TargetTransformInfo::TCC_Basic;
     return TargetTransformInfo::TCC_Expensive;
   }
 
-  unsigned getInliningThresholdMultiplier() { return 1; }
+  unsigned getInliningThresholdMultiplier() const { return 1; }
   unsigned adjustInliningThreshold(const CallBase *CB) { return 0; }
+  unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
+    return 0;
+  }
 
-  int getInlinerVectorBonusPercent() { return 150; }
+  int getInlinerVectorBonusPercent() const { return 150; }
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
                                OptimizationRemarkEmitter *ORE) {
     // This unrolling functionality is target independent, but to provide some
     // motivation for its intended use, for x86:
 
     // According to the Intel 64 and IA-32 Architectures Optimization Reference

llvm/lib/Analysis/InlineCost.cpp

void onFinalizeSwitch(unsigned JumpTableSize,
 
     addCost(SwitchCost);
   }
   void onMissedSimplification() override { addCost(InstrCost); }
 
   void onInitializeSROAArg(AllocaInst *Arg) override {
     assert(Arg != nullptr &&
            "Should not initialize SROA costs for null value.");
-    SROAArgCosts[Arg] = 0;
+    SROACostSavings += SROAArgCosts[Arg] =
+        TTI.getCallerAllocaCost(&CandidateCall, Arg);
   }
 
   void onAggregateSROAUse(AllocaInst *SROAArg) override {
     auto CostIt = SROAArgCosts.find(SROAArg);
     assert(CostIt != SROAArgCosts.end() &&
            "expected this argument to have a cost");
     CostIt->second += InstrCost;
     SROACostSavings += InstrCost;

llvm/lib/Analysis/TargetTransformInfo.cpp

unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
   return TTIImpl->getInliningThresholdMultiplier();
 }
 
 unsigned
 TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
   return TTIImpl->adjustInliningThreshold(CB);
 }
 
+unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB,
+                                                  const AllocaInst *AI) const {
+  return TTIImpl->getCallerAllocaCost(CB, AI);
+}
+
 int TargetTransformInfo::getInlinerVectorBonusPercent() const {
   return TTIImpl->getInlinerVectorBonusPercent();
 }
 
 InstructionCost
 TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr,
                                 ArrayRef<const Value *> Operands,
                                 TTI::TargetCostKind CostKind) const {

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = std::nullopt);
 
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
-  unsigned getInliningThresholdMultiplier() { return 11; }
+  unsigned getInliningThresholdMultiplier() const { return 11; }
   unsigned adjustInliningThreshold(const CallBase *CB) const;
+  unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const;
 
-  int getInlinerVectorBonusPercent() { return 0; }
+  int getInlinerVectorBonusPercent() const { return 0; }
 
   InstructionCost getArithmeticReductionCost(
       unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
       TTI::TargetCostKind CostKind);
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind);
   InstructionCost getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
                                          bool IsUnsigned, FastMathFlags FMF,
                                          TTI::TargetCostKind CostKind);
 };
 
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,
   // not model any storage costs.
   adjustThreshold += std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) *
                      *ArgStackCost.getValue() * InlineConstants::getInstrCost();
   adjustThreshold += std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) *
                      *ArgStackCost.getValue() * InlineConstants::getInstrCost();
   return adjustThreshold;
 }
 
-unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
+static size_t getTotalAllocaSize(const CallBase *CB, const DataLayout &DL) {
   // If we have a pointer to private array passed into a function
   // it will not be optimized out, leaving scratch usage.
   // Increase the inline threshold to allow inlining in this case.
-  unsigned adjustThreshold = 0;
+  // The bigger the private array the bigger the bonus.
   uint64_t AllocaSize = 0;
   SmallPtrSet<const AllocaInst *, 8> AIVisited;
   for (Value *PtrArg : CB->args()) {
     PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType());
     if (!Ty || (Ty->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS &&
                 Ty->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS))
       continue;
 
     PtrArg = getUnderlyingObject(PtrArg);
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(PtrArg)) {
       if (!AI->isStaticAlloca() || !AIVisited.insert(AI).second)
         continue;
       AllocaSize += DL.getTypeAllocSize(AI->getAllocatedType());
-      // If the amount of stack memory is excessive we will not be able
-      // to get rid of the scratch anyway, bail out.
-      if (AllocaSize > ArgAllocaCutoff) {
-        AllocaSize = 0;
-        break;
-      }
-    }
-  }
-  adjustThreshold +=
-      adjustInliningThresholdUsingCallee(CB, TLI, this);
-  adjustThreshold += AllocaSize ? ArgAllocaCost : AllocaSize;
-  return adjustThreshold;
+  return AllocaSize;
+
+unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
+  unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, this);
+  size_t AllocaSize = getTotalAllocaSize(CB, DL);
+  if (AllocaSize > 0) {
+    Threshold += ArgAllocaCost;
+  }
+  return Threshold;
+}
+
+unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
+                                         const AllocaInst *AI) const {
+  auto AllocaSize = getTotalAllocaSize(CB, DL);
+  if (AllocaSize <= ArgAllocaCutoff) {
+    // the inline threshold bonus has been given already by
+    // adjustInliningThreshold
+    return 0;
+  }
+
+  // give an inline threshold bonus depending on the size of the alloca that is
+  // being optimized by SROA. the bigger the array is, the better chances we
+  // have to avoid scratch memory
+  auto ArgAllocaSize = DL.getTypeAllocSize(AI->getAllocatedType());
+  unsigned ThresholdBonus =
+      (ArgAllocaCost * getInliningThresholdMultiplier() * ArgAllocaSize) /
+      AllocaSize;
+
+  // awkwardly, this bonus gets multiplied by the single-bb-bonus and the
+  // vector-bonus
+  return ThresholdBonus;
 }
 
 void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                          TTI::UnrollingPreferences &UP,
                                          OptimizationRemarkEmitter *ORE) {
   CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
 }
 

llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h

public:
   // no incompatible functions in PTX, ptxas will throw errors in such cases.
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const {
     return true;
   }
 
   // Increase the inlining cost threshold by a factor of 11, reflecting that
   // calls are particularly expensive in NVPTX.
-  unsigned getInliningThresholdMultiplier() { return 11; }
+  unsigned getInliningThresholdMultiplier() const { return 11; }
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
       const Instruction *CxtI = nullptr);
 

llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h

 public:
   explicit SystemZTTIImpl(const SystemZTargetMachine *TM, const Function &F)
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
   /// \name Scalar TTI Implementations
   /// @{
 
-  unsigned getInliningThresholdMultiplier() { return 3; }
+  unsigned getInliningThresholdMultiplier() const { return 3; }
   unsigned adjustInliningThreshold(const CallBase *CB) const;
 
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
                                 TTI::TargetCostKind CostKind);
 
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,

llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll

if.then:                                          ; preds = %entry
   br label %if.end
 
 if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
 define coldcc void @foo_private_ptr2(ptr addrspace(5) nocapture %p1, ptr addrspace(5) nocapture %p2) {
 entry:
+  call void @forbid_sroa(ptr addrspace(5) %p1)
+  call void @forbid_sroa(ptr addrspace(5) %p2)
   %tmp1 = load float, ptr addrspace(5) %p1, align 4
   %cmp = fcmp ogt float %tmp1, 1.000000e+00
   br i1 %cmp, label %if.then, label %if.end
 
 if.then:
   %div = fdiv float 2.000000e+00, %tmp1
   store float %div, ptr addrspace(5) %p2, align 4
   br label %if.end

 bb.2:
   %c = call float @sin_wrapper(float 1.0)
   store float %c, ptr addrspace(1) %a
   ret void
 }
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare float @_Z3sinf(float) #1
+declare void @forbid_sroa(ptr addrspace(5) nocapture %p)
 
 attributes #0 = { noinline }
 attributes #1 = { nounwind readnone }

llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-alloca-argument-cost.ll

-; RUN: opt -mtriple=amdgcn--amdhsa -S -passes=inline -inline-threshold=0 -debug-only=inline-cost < %s 2>&1 | FileCheck %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -passes=inline -inline-threshold=0 -debug-only=inline-cost %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
 target datalayout = "A5"
 
 ; Verify we are properly adding cost of the -amdgpu-inline-arg-alloca-cost to the threshold.
+; Awkwardly, the adjusted threshold is multiplied by the single-basic-block bonus in .
 
+define void @local_access_only(ptr addrspace(5) %p, i32 %idx) {
+  %arrayidx = getelementptr inbounds [64 x float], ptr addrspace(5) %p, i32 0, i32 %idx
+  %value = load float, ptr addrspace(5) %arrayidx
+  store float %value , ptr addrspace(5) %arrayidx, align 4
+  ret void
+}
+
+; below the cutoff, the alloca cost is 0, and only the cost of the instructions saved by sroa is counted
+; CHECK: Analyzing call of local_access_only... (caller:test_inliner_sroa_single_below_cutoff)
+; CHECK: NumAllocaArgs: 1
+; CHECK: SROACostSavings: 10
+; CHECK: SROACostSavingsLost: 0
+; CHECK: Threshold: 66000
+define amdgpu_kernel void @test_inliner_sroa_single_below_cutoff(ptr addrspace(1) %a, i32 %n) {
+entry:
+  %pvt_arr = alloca [64 x float], align 4, addrspace(5)
+  call void @local_access_only(ptr addrspace(5) %pvt_arr, i32 4)
+  ret void
+}
+
+; above the cutoff, attribute a cost to the alloca
+; CHECK: Analyzing call of local_access_only... (caller:test_inliner_sroa_single_above_cutoff)
 ; CHECK: NumAllocaArgs: 1
+; CHECK: SROACostSavings: 44010
+; CHECK: SROACostSavingsLost: 0
+; CHECK: Threshold: 66000
+define amdgpu_kernel void @test_inliner_sroa_single_above_cutoff(ptr addrspace(1) %a, i32 %n) {
+entry:
+  %pvt_arr = alloca [65 x float], align 4, addrspace(5)
+  call void @local_access_only(ptr addrspace(5) %pvt_arr, i32 4)
+  ret void
+}
+
+define void @use_first_externally(ptr addrspace(5) %p1, ptr addrspace(5) %p2) {
+  call void @external(ptr addrspace(5) %p1)
+  %arrayidx = getelementptr inbounds [64 x float], ptr addrspace(5) %p2, i32 0, i32 7
+  %value = load float, ptr addrspace(5) %arrayidx
+  store float %value , ptr addrspace(5) %arrayidx, align 4
+  ret void
+}
+
+define void @use_both_externally(ptr addrspace(5) %p1, ptr addrspace(5) %p2) {
+  call void @external(ptr addrspace(5) %p1)
+  call void @external(ptr addrspace(5) %p2)
+  ret void
+}
+
+; CHECK: Analyzing call of use_first_externally... (caller:test_inliner_sroa_double)
+; CHECK: NumAllocaArgs: 2
+; CHECK: SROACostSavings: 21671
+; CHECK: SROACostSavingsLost: 22338
 ; CHECK: Threshold: 66000
+define amdgpu_kernel void @test_inliner_sroa_double() {
+entry:
+  %pvt_arr1 = alloca [33 x float], align 4, addrspace(5)
+  %pvt_arr2 = alloca [32 x float], align 4, addrspace(5)
+  call void @use_first_externally(ptr addrspace(5) %pvt_arr1, ptr addrspace(5) %pvt_arr2)
+  ret void
+}
 
-define void @use_private_ptr_arg(ptr addrspace(5) nocapture %p) {
+; CHECK: Analyzing call of use_both_externally... (caller:test_inliner_no_sroa)
+; CHECK: NumAllocaArgs: 2
+; CHECK: SROACostSavings: 0
+; CHECK: SROACostSavingsLost: 43999
+; CHECK: Threshold: 66000
+define amdgpu_kernel void @test_inliner_no_sroa() {
+entry:
+  %pvt_arr1 = alloca [33 x float], align 4, addrspace(5)
+  %pvt_arr2 = alloca [32 x float], align 4, addrspace(5)
+  call void @use_both_externally(ptr addrspace(5) %pvt_arr1, ptr addrspace(5) %pvt_arr2)
   ret void
 }
 
-define amdgpu_kernel void @test_inliner_pvt_ptr(ptr addrspace(1) nocapture %a, i32 %n) {
+; CHECK: Analyzing call of use_both_externally... (caller:test_inliner_no_alloc)
+; CHECK: NumAllocaArgs: 0
+; CHECK: SROACostSavings: 0
+; CHECK: SROACostSavingsLost: 0
+; CHECK: Threshold: 0
+define amdgpu_kernel void @test_inliner_no_alloc(ptr addrspace(5) %a, ptr addrspace(5) %b) {
 entry:
-  %pvt_arr = alloca [64 x float], align 4, addrspace(5)
-  call void @use_private_ptr_arg(ptr addrspace(5) %pvt_arr)
+  call void @use_both_externally(ptr addrspace(5) %a, ptr addrspace(5) %b)
   ret void
 }
+
+declare void @external(ptr addrspace(5) %p)
 No newline at end of file