[VPlan] Add & use VPValue operands for VPWidenCallRecipe (NFC).

[fhahn]

Summary

This patch adds VPValue versions for the arguments of the call to
VPWidenCallRecipe and uses them during code-generation.

Similar to D76373 this reduces ingredient def-use usage by ILV as
a step towards full VPlan-based def-use relations.

[gilr]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

6931

Have VPWidenCallRecipe() take an iterator_range instead of ArrayRef, saving the temporary SmallVector?

[fhahn]

use iterator_range in VPWidenCallRecipe and VPUser constructors

[fhahn]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

6931

The way the types stack up with iterator_range and map_iterator we would need to add template version of the VPWidenCallRecipe and VPUser constructors. I think that would make sense, to provide more convenience to users of those classes. I;ve updated the patch. What do you think?

[gilr]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

6931

LGTM, thanks!

llvm/lib/Transforms/Vectorize/VPlanValue.h

25

Lexical order

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

public:
   /// Create a new empty loop. Unlink the old loop and connect the new one.
   /// Return the pre-header block of the new loop.
   BasicBlock *createVectorizedLoopSkeleton();
 
   /// Widen a single instruction within the innermost loop.
   void widenInstruction(Instruction &I);
 
   /// Widen a single call instruction within the innermost loop.
-  void widenCallInstruction(CallInst &I);
+  void widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+                            VPTransformState &State);
 
   /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
   void fixVectorizedLoop();
 
   // Return true if any runtime check is added.
   bool areSafetyChecksAdded() { return AddedSafetyChecks; }
 
   /// A type for vectorized values in the new loop. Each value from the
void InnerLoopVectorizer::widenInstruction(Instruction &I) {
   }
   default:
     // This instruction is not vectorized by simple widening.
     LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
     llvm_unreachable("Unhandled instruction!");
   } // end of switch.
 }
 
-void InnerLoopVectorizer::widenCallInstruction(CallInst &I) {
+void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+                                               VPTransformState &State) {
   // Ignore dbg intrinsics.
   // TODO: Debug intrinsics should be skipped/handled during VPlan construction
   // rather than dropping them here.
   if (isa<DbgInfoIntrinsic>(I))
     return;
   setDebugLocFromInst(Builder, &I);
 
   Module *M = I.getParent()->getParent()->getParent();
void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
   unsigned CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
   bool UseVectorIntrinsic =
       ID && Cost->getVectorIntrinsicCost(CI, VF) <= CallCost;
   assert((UseVectorIntrinsic || !NeedToScalarize) &&
          "Instruction should be scalarized elsewhere.");
 
   for (unsigned Part = 0; Part < UF; ++Part) {
     SmallVector<Value *, 4> Args;
-    for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) {
-      Value *Arg = CI->getArgOperand(i);
+    for (auto &I : enumerate(ArgOperands.operands())) {
       // Some intrinsics have a scalar argument - don't replace it with a
       // vector.
-      if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i))
-        Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part);
+      Value *Arg;
+      if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
+        Arg = State.get(I.value(), Part);
+      else
+        Arg = State.get(I.value(), {0, 0});
       Args.push_back(Arg);
     }
 
     Function *VectorF;
     if (UseVectorIntrinsic) {
       // Use vector version of the intrinsic.
       Type *TysForDecl[] = {CI->getType()};
       if (VF > 1)
for (unsigned In = 0; In < NumIncoming; In++) {
     assert((EdgeMask || NumIncoming == 1) &&
            "Multiple predecessors with one having a full mask");
     if (EdgeMask)
       Masks.push_back(EdgeMask);
   }
   return new VPBlendRecipe(Phi, Masks);
 }
 
-VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(Instruction *I,
-                                                   VFRange &Range) {
+VPWidenCallRecipe *
+VPRecipeBuilder::tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan) {
 
   bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
 
   CallInst *CI = dyn_cast<CallInst>(I);
   if (IsPredicated || !CI)
     return nullptr;
 
bool UseVectorIntrinsic =
         ID && CM.getVectorIntrinsicCost(CI, VF) <= CallCost;
     return UseVectorIntrinsic || !NeedToScalarize;
   };
 
   if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
     return nullptr;
 
   // Success: widen this call.
-  return new VPWidenCallRecipe(*CI);
+  auto VPValues = map_range(CI->arg_operands(), [&Plan](Value *Op) {
+    return Plan.getOrAddVPValue(Op);
+  });
+
+  return new VPWidenCallRecipe(*CI, VPValues);
 }
 
 VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) {
   bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
 
   if (IsPredicated)
     return nullptr;
 

 }
 
 bool VPRecipeBuilder::tryToCreateRecipe(Instruction *Instr, VFRange &Range,
                                         VPlanPtr &Plan, VPBasicBlock *VPBB) {
   VPRecipeBase *Recipe = nullptr;
 
   // First, check for specific widening recipes that deal with calls, memory
   // operations, inductions and Phi nodes.
-  if ((Recipe = tryToWidenCall(Instr, Range)) ||
+  if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) ||
       (Recipe = tryToWidenMemory(Instr, Range, Plan)) ||
       (Recipe = tryToOptimizeInduction(Instr, Range)) ||
       (Recipe = tryToBlend(Instr, Plan)) ||
       (isa<PHINode>(Instr) &&
        (Recipe = new VPWidenPHIRecipe(cast<PHINode>(Instr))))) {
     setRecipe(Instr, Recipe);
     VPBB->appendRecipe(Recipe);
     return true;
void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
   O << "\\l\"";
   for (unsigned i = 0; i < IG->getFactor(); ++i)
     if (Instruction *I = IG->getMember(i))
       O << " +\n"
         << Indent << "\"  " << VPlanIngredient(I) << " " << i << "\\l\"";
 }
 
 void VPWidenCallRecipe::execute(VPTransformState &State) {
-  State.ILV->widenCallInstruction(Ingredient);
+  State.ILV->widenCallInstruction(Ingredient, User, State);
 }
 
 void VPWidenRecipe::execute(VPTransformState &State) {
   State.ILV->widenInstruction(Ingredient);
 }
 
 void VPWidenGEPRecipe::execute(VPTransformState &State) {
   State.ILV->widenGEP(GEP, State.UF, State.VF, IsPtrLoopInvariant,

llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

public:
   /// Handle non-loop phi nodes. Currently all such phi nodes are turned into
   /// a sequence of select instructions as the vectorizer currently performs
   /// full if-conversion.
   VPBlendRecipe *tryToBlend(Instruction *I, VPlanPtr &Plan);
 
   /// Handle call instruction. If \p I is a call that can be widened for \p
   /// Range.Start, return a new VPWidenCallRecipe. Range.End may be decreased to
   /// ensure same decision from \p Range.Start to \p Range.End.
-  VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range);
+  VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range,
+                                    VPlan &Plan);
 
   /// Check if \p I can be widened within the given VF \p Range. If \p I can be
   /// widened for \p Range.Start, build a new VPWidenRecipe and return it.
   /// Range.End may be decreased to ensure same decision from \p Range.Start to
   /// \p Range.End.
   VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range);
 
   /// Create a replicating region for instruction \p I that requires

llvm/lib/Transforms/Vectorize/VPlan.h

 };
 
 /// A recipe for widening Call instructions.
 class VPWidenCallRecipe : public VPRecipeBase {
 private:
   /// Hold the call to be widened.
   CallInst &Ingredient;
 
+  /// Hold VPValues for the arguments of the call.
+  VPUser User;
+
 public:
-  VPWidenCallRecipe(CallInst &I) : VPRecipeBase(VPWidenCallSC), Ingredient(I) {}
+  template <typename IterT>
+  VPWidenCallRecipe(CallInst &I, iterator_range<IterT> CallArguments)
+      : VPRecipeBase(VPWidenCallSC), Ingredient(I), User(CallArguments) {}
 
   ~VPWidenCallRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPRecipeBase *V) {
     return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC;
   }
 

llvm/lib/Transforms/Vectorize/VPlanValue.h

 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H
 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 
 // Forward declarations.
 class VPUser;
for (VPValue *Operand : Operands)
       addOperand(Operand);
   }
 
 public:
   VPUser() : VPValue(VPValue::VPUserSC) {}
   VPUser(ArrayRef<VPValue *> Operands) : VPUser(VPValue::VPUserSC, Operands) {}
   VPUser(std::initializer_list<VPValue *> Operands)
       : VPUser(ArrayRef<VPValue *>(Operands)) {}
+  template <typename IterT>
+  VPUser(iterator_range<IterT> Operands) : VPValue(VPValue::VPUserSC) {
+    for (VPValue *Operand : Operands)
+      addOperand(Operand);
+  }
+
   VPUser(const VPUser &) = delete;
   VPUser &operator=(const VPUser &) = delete;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPValue *V) {
     return V->getVPValueID() >= VPUserSC &&
            V->getVPValueID() <= VPInstructionSC;
   }