[InlineCost] separate stats from CallAnalyzer into InliningStats

[fedor.sergeev]

Summary

Inline cost analysis done by CallAnalyzer counts various stats about function body
being analyzed. These stats can be useful for later analysis, say, to give additional
bonuses for specific constructs, to use them in prioritization of inlining decisions
when working on a tight budget (a pressing issue for JIT compilers).

These stats are exposed outside of getInlineCost via a callback passed through Params.

[greened]

llvm/lib/Analysis/InlineCost.cpp

2105

Is there a particular reason you chose to do this via callback rather than an optional passed result argument?

llvm/include/llvm/Analysis/InlineCost.h

struct InlineResult {
   const char *message = nullptr;
   InlineResult(bool result, const char *message = nullptr)
       : message(result ? nullptr : (message ? message : "cost > threshold")) {}
   InlineResult(const char *message = nullptr) : message(message) {}
   operator bool() const { return !message; }
   operator const char *() const { return message; }
 };
 
+/// Keeps a bunch of stats about the interesting simplification constructs found
+/// during inline-cost analysis so we can use them outside of InlineCost call
+/// analysis.
+struct InlineCostStats {
+  unsigned NumConstantArgs = 0;
+  unsigned NumConstantOffsetPtrArgs = 0;
+  unsigned NumAllocaArgs = 0;
+  unsigned NumConstantPtrCmps = 0;
+  unsigned NumConstantPtrDiffs = 0;
+  unsigned NumInstructionsSimplified = 0;
+  unsigned NumInstructions = 0;
+  unsigned NumVectorInstructions = 0;
+
+  LLVM_DUMP_METHOD void dump();
+};
+
 /// Thresholds to tune inline cost analysis. The inline cost analysis decides
 /// the condition to apply a threshold and applies it. Otherwise,
 /// DefaultThreshold is used. If a threshold is Optional, it is applied only
 /// when it has a valid value. Typically, users of inline cost analysis
 /// obtain an InlineParams object through one of the \c getInlineParams methods
 /// and pass it to \c getInlineCost. Some specialized versions of inliner
 /// (such as the pre-inliner) might have custom logic to compute \c InlineParams
 /// object.
struct InlineParams {
   /// entry.
   Optional<int> LocallyHotCallSiteThreshold;
 
   /// Threshold to use when the callsite is considered cold.
   Optional<int> ColdCallSiteThreshold;
 
   /// Compute inline cost even when the cost has exceeded the threshold.
   Optional<bool> ComputeFullInlineCost;
+
+  /// Callback to pass inline cost stats to the caller of getInlineCost.
+  Optional<function_ref<void(InlineCostStats)>> PassInlineCostStats;
 };
 
 /// Generate the parameters to tune the inline cost analysis based only on the
 /// commandline options.
 InlineParams getInlineParams();
 
 /// Generate the parameters to tune the inline cost analysis based on command
 /// line options. If -inline-threshold option is not explicitly passed,

llvm/lib/Analysis/InlineCost.cpp

class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool ContainsNoDuplicateCall = false;
   bool HasReturn = false;
   bool HasIndirectBr = false;
   bool HasUninlineableIntrinsic = false;
   bool InitsVargArgs = false;
 
   /// Number of bytes allocated statically by the callee.
   uint64_t AllocatedSize = 0;
-  unsigned NumInstructions = 0;
-  unsigned NumVectorInstructions = 0;
 
   /// Bonus to be applied when percentage of vector instructions in callee is
   /// high (see more details in updateThreshold).
   int VectorBonus = 0;
   /// Bonus to be applied when the callee has only one reachable basic block.
   int SingleBBBonus = 0;
 
   /// While we walk the potentially-inlined instructions, we build up and
class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   /// allocas on the caller stack which could be simplified through SROA.
   DenseMap<Value *, Value *> SROAArgValues;
 
   /// The mapping of caller Alloca values to their accumulated cost savings. If
   /// we have to disable SROA for one of the allocas, this tells us how much
   /// cost must be added.
   DenseMap<Value *, int> SROAArgCosts;
 
+  unsigned SROACostSavings = 0;
+  unsigned SROACostSavingsLost = 0;
+
   /// Keep track of values which map to a pointer base and constant offset.
   DenseMap<Value *, std::pair<Value *, APInt>> ConstantOffsetPtrs;
 
   /// Keep track of dead blocks due to the constant arguments.
   SetVector<BasicBlock *> DeadBlocks;
 
   /// The mapping of the blocks to their known unique successors due to the
   /// constant arguments.
   DenseMap<BasicBlock *, BasicBlock *> KnownSuccessors;
 
   /// Model the elimination of repeated loads that is expected to happen
   /// whenever we simplify away the stores that would otherwise cause them to be
   /// loads.
   bool EnableLoadElimination;
   SmallPtrSet<Value *, 16> LoadAddrSet;
   int LoadEliminationCost = 0;
 
+  InlineCostStats InliningStats;
+
   // Custom simplification helper routines.
   bool isAllocaDerivedArg(Value *V);
   bool lookupSROAArgAndCost(Value *V, Value *&Arg,
                             DenseMap<Value *, int>::iterator &CostIt);
   void disableSROA(DenseMap<Value *, int>::iterator CostIt);
   void disableSROA(Value *V);
   void findDeadBlocks(BasicBlock *CurrBB, BasicBlock *NextBB);
   void accumulateSROACost(DenseMap<Value *, int>::iterator CostIt,
CallAnalyzer(const TargetTransformInfo &TTI,
                               Params.ComputeFullInlineCost || ORE),
         EnableLoadElimination(true) {}
 
   InlineResult analyzeCall(CallBase &Call);
 
   int getThreshold() { return Threshold; }
   int getCost() { return Cost; }
 
-  // Keep a bunch of stats about the cost savings found so we can print them
-  // out when debugging.
-  unsigned NumConstantArgs = 0;
-  unsigned NumConstantOffsetPtrArgs = 0;
-  unsigned NumAllocaArgs = 0;
-  unsigned NumConstantPtrCmps = 0;
-  unsigned NumConstantPtrDiffs = 0;
-  unsigned NumInstructionsSimplified = 0;
-  unsigned SROACostSavings = 0;
-  unsigned SROACostSavingsLost = 0;
+  InlineCostStats getInlineCostStats() { return InliningStats; }
 
   void dump();
 };
 
 } // namespace
 
 /// Test whether the given value is an Alloca-derived function argument.
 bool CallAnalyzer::isAllocaDerivedArg(Value *V) {
if (LHSBase) {
     std::tie(RHSBase, RHSOffset) = ConstantOffsetPtrs.lookup(RHS);
     if (RHSBase && LHSBase == RHSBase) {
       // We have common bases, fold the icmp to a constant based on the
       // offsets.
       Constant *CLHS = ConstantInt::get(LHS->getContext(), LHSOffset);
       Constant *CRHS = ConstantInt::get(RHS->getContext(), RHSOffset);
       if (Constant *C = ConstantExpr::getICmp(I.getPredicate(), CLHS, CRHS)) {
         SimplifiedValues[&I] = C;
-        ++NumConstantPtrCmps;
+        ++InliningStats.NumConstantPtrCmps;
         return true;
       }
     }
   }
 
   // If the comparison is an equality comparison with null, we can simplify it
   // if we know the value (argument) can't be null
   if (I.isEquality() && isa<ConstantPointerNull>(I.getOperand(1)) &&
if (LHSBase) {
     std::tie(RHSBase, RHSOffset) = ConstantOffsetPtrs.lookup(RHS);
     if (RHSBase && LHSBase == RHSBase) {
       // We have common bases, fold the subtract to a constant based on the
       // offsets.
       Constant *CLHS = ConstantInt::get(LHS->getContext(), LHSOffset);
       Constant *CRHS = ConstantInt::get(RHS->getContext(), RHSOffset);
       if (Constant *C = ConstantExpr::getSub(CLHS, CRHS)) {
         SimplifiedValues[&I] = C;
-        ++NumConstantPtrDiffs;
+        ++InliningStats.NumConstantPtrDiffs;
         return true;
       }
     }
   }
 
   // Otherwise, fall back to the generic logic for simplifying and handling
   // instructions.
   return Base::visitSub(I);
for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
     // hack around it here.
     if (isa<DbgInfoIntrinsic>(I))
       continue;
 
     // Skip ephemeral values.
     if (EphValues.count(&*I))
       continue;
 
-    ++NumInstructions;
+    ++InliningStats.NumInstructions;
     if (isa<ExtractElementInst>(I) || I->getType()->isVectorTy())
-      ++NumVectorInstructions;
+      ++InliningStats.NumVectorInstructions;
 
     // If the instruction simplified to a constant, there is no cost to this
     // instruction. Visit the instructions using our InstVisitor to account for
     // all of the per-instruction logic. The visit tree returns true if we
     // consumed the instruction in any way, and false if the instruction's base
     // cost should count against inlining.
     if (Base::visit(&*I))
-      ++NumInstructionsSimplified;
+      ++InliningStats.NumInstructionsSimplified;
     else
       addCost(InlineConstants::InstrCost);
 
     using namespace ore;
     // If the visit this instruction detected an uninlinable pattern, abort.
     InlineResult IR;
     if (IsRecursiveCall)
       IR = "recursive";
InlineResult CallAnalyzer::analyzeCall(CallBase &Call) {
 
   // We want to more aggressively inline vector-dense kernels, so up the
   // threshold, and we'll lower it if the % of vector instructions gets too
   // low. Note that these bonuses are some what arbitrary and evolved over time
   // by accident as much as because they are principled bonuses.
   //
   // FIXME: It would be nice to remove all such bonuses. At least it would be
   // nice to base the bonus values on something more scientific.
-  assert(NumInstructions == 0);
-  assert(NumVectorInstructions == 0);
+  assert(InliningStats.NumInstructions == 0);
+  assert(InliningStats.NumVectorInstructions == 0);
 
   // Update the threshold based on callsite properties
   updateThreshold(Call, F);
 
   // While Threshold depends on commandline options that can take negative
   // values, we want to enforce the invariant that the computed threshold and
   // bonuses are non-negative.
   assert(Threshold >= 0);
if (ConstantInt *C = stripAndComputeInBoundsConstantOffsets(PtrArg)) {
 
       // We can SROA any pointer arguments derived from alloca instructions.
       if (isa<AllocaInst>(PtrArg)) {
         SROAArgValues[&*FAI] = PtrArg;
         SROAArgCosts[PtrArg] = 0;
       }
     }
   }
-  NumConstantArgs = SimplifiedValues.size();
-  NumConstantOffsetPtrArgs = ConstantOffsetPtrs.size();
-  NumAllocaArgs = SROAArgValues.size();
+  InliningStats.NumConstantArgs = SimplifiedValues.size();
+  InliningStats.NumConstantOffsetPtrArgs = ConstantOffsetPtrs.size();
+  InliningStats.NumAllocaArgs = SROAArgValues.size();
 
   // FIXME: If a caller has multiple calls to a callee, we end up recomputing
   // the ephemeral values multiple times (and they're completely determined by
   // the callee, so this is purely duplicate work).
   SmallPtrSet<const Value *, 32> EphValues;
   CodeMetrics::collectEphemeralValues(&F, &GetAssumptionCache(F), EphValues);
 
   // The worklist of live basic blocks in the callee *after* inlining. We avoid
for (Loop *L : LI) {
       NumLoops++;
     }
     addCost(NumLoops * InlineConstants::CallPenalty);
   }
 
   // We applied the maximum possible vector bonus at the beginning. Now,
   // subtract the excess bonus, if any, from the Threshold before
   // comparing against Cost.
-  if (NumVectorInstructions <= NumInstructions / 10)
+  if (InliningStats.NumVectorInstructions <= InliningStats.NumInstructions / 10)
     Threshold -= VectorBonus;
-  else if (NumVectorInstructions <= NumInstructions / 2)
+  else if (InliningStats.NumVectorInstructions <=
+           InliningStats.NumInstructions / 2)
     Threshold -= VectorBonus/2;
 
   return Cost < std::max(1, Threshold);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-/// Dump stats about this call's analysis.
 LLVM_DUMP_METHOD void CallAnalyzer::dump() {
 #define DEBUG_PRINT_STAT(x) dbgs() << "      " #x ": " << x << "\n"
+  InliningStats.dump();
+  DEBUG_PRINT_STAT(ContainsNoDuplicateCall);
+  DEBUG_PRINT_STAT(SROACostSavings);
+  DEBUG_PRINT_STAT(SROACostSavingsLost);
+  DEBUG_PRINT_STAT(LoadEliminationCost);
+  DEBUG_PRINT_STAT(Cost);
+  DEBUG_PRINT_STAT(Threshold);
+#undef DEBUG_PRINT_STAT
+}
+
+/// Dump stats about this call's analysis.
+LLVM_DUMP_METHOD void InlineCostStats::dump() {
+#define DEBUG_PRINT_STAT(x) dbgs() << "      " #x ": " << x << "\n"
   DEBUG_PRINT_STAT(NumConstantArgs);
   DEBUG_PRINT_STAT(NumConstantOffsetPtrArgs);
   DEBUG_PRINT_STAT(NumAllocaArgs);
   DEBUG_PRINT_STAT(NumConstantPtrCmps);
   DEBUG_PRINT_STAT(NumConstantPtrDiffs);
   DEBUG_PRINT_STAT(NumInstructionsSimplified);
   DEBUG_PRINT_STAT(NumInstructions);
-  DEBUG_PRINT_STAT(SROACostSavings);
-  DEBUG_PRINT_STAT(SROACostSavingsLost);
-  DEBUG_PRINT_STAT(LoadEliminationCost);
-  DEBUG_PRINT_STAT(ContainsNoDuplicateCall);
-  DEBUG_PRINT_STAT(Cost);
-  DEBUG_PRINT_STAT(Threshold);
 #undef DEBUG_PRINT_STAT
 }
 #endif
 
 /// Test that there are no attribute conflicts between Caller and Callee
 ///        that prevent inlining.
 static bool functionsHaveCompatibleAttributes(Function *Caller,
                                               Function *Callee,
InlineCost llvm::getInlineCost(
 
   LLVM_DEBUG(llvm::dbgs() << "      Analyzing call of " << Callee->getName()
                           << "... (caller:" << Caller->getName() << ")\n");
 
   CallAnalyzer CA(CalleeTTI, GetAssumptionCache, GetBFI, PSI, ORE, *Callee,
                   Call, Params);
   InlineResult ShouldInline = CA.analyzeCall(Call);
 
+  if (Params.PassInlineCostStats)
+    (*Params.PassInlineCostStats)(CA.getInlineCostStats());
+
   LLVM_DEBUG(CA.dump());
 
   // Check if there was a reason to force inlining or no inlining.
   if (!ShouldInline && CA.getCost() < CA.getThreshold())
     return InlineCost::getNever(ShouldInline.message);
   if (ShouldInline && CA.getCost() >= CA.getThreshold())
     return InlineCost::getAlways("empty function");