[VPlan] Add & use VPValue operands for VPWidenCallRecipe (NFC).

[fhahn]

Summary

This patch adds VPValue versions for the arguments of the call to
VPWidenCallRecipe and uses them during code-generation.

Similar to D76373 this reduces ingredient def-use usage by ILV as
a step towards full VPlan-based def-use relations.

[gilr]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

6935

Have VPWidenCallRecipe() take an iterator_range instead of ArrayRef, saving the temporary SmallVector?

[fhahn]

use iterator_range in VPWidenCallRecipe and VPUser constructors

[fhahn]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

6935

The way the types stack up with iterator_range and map_iterator we would need to add template version of the VPWidenCallRecipe and VPUser constructors. I think that would make sense, to provide more convenience to users of those classes. I;ve updated the patch. What do you think?

[gilr]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

6935

LGTM, thanks!

llvm/lib/Transforms/Vectorize/VPlanValue.h

25 ↗

(On Diff #256071)

Lexical order

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

public:
   BasicBlock *createVectorizedLoopSkeleton();
 
   /// Widen a single instruction within the innermost loop.
   void widenInstruction(Instruction &I, VPUser &Operands,
                         VPTransformState &State,
                         std::function<Value *(unsigned)> GetUnderlyingValue);
 
   /// Widen a single call instruction within the innermost loop.
-  void widenCallInstruction(CallInst &I);
+  void widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+                            VPTransformState &State);
 
   /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
   void fixVectorizedLoop();
 
   // Return true if any runtime check is added.
   bool areSafetyChecksAdded() { return AddedSafetyChecks; }
 
   /// A type for vectorized values in the new loop. Each value from the
void InnerLoopVectorizer::widenInstruction(
   }
   default:
     // This instruction is not vectorized by simple widening.
     LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
     llvm_unreachable("Unhandled instruction!");
   } // end of switch.
 }
 
-void InnerLoopVectorizer::widenCallInstruction(CallInst &I) {
+void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+                                               VPTransformState &State) {
   // Ignore dbg intrinsics.
   // TODO: Debug intrinsics should be skipped/handled during VPlan construction
   // rather than dropping them here.
   if (isa<DbgInfoIntrinsic>(I))
     return;
   setDebugLocFromInst(Builder, &I);
 
   Module *M = I.getParent()->getParent()->getParent();
void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
   unsigned CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
   bool UseVectorIntrinsic =
       ID && Cost->getVectorIntrinsicCost(CI, VF) <= CallCost;
   assert((UseVectorIntrinsic || !NeedToScalarize) &&
          "Instruction should be scalarized elsewhere.");
 
   for (unsigned Part = 0; Part < UF; ++Part) {
     SmallVector<Value *, 4> Args;
-    for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) {
-      Value *Arg = CI->getArgOperand(i);
+    for (auto &I : enumerate(ArgOperands.operands())) {
       // Some intrinsics have a scalar argument - don't replace it with a
       // vector.
-      if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i))
-        Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part);
+      Value *Arg;
+      if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
+        Arg = State.get(I.value(), Part);
+      else
+        Arg = State.get(I.value(), {0, 0});
       Args.push_back(Arg);
     }
 
     Function *VectorF;
     if (UseVectorIntrinsic) {
       // Use vector version of the intrinsic.
       Type *TysForDecl[] = {CI->getType()};
       if (VF > 1)
for (unsigned In = 0; In < NumIncoming; In++) {
     assert((EdgeMask || NumIncoming == 1) &&
            "Multiple predecessors with one having a full mask");
     if (EdgeMask)
       Masks.push_back(EdgeMask);
   }
   return new VPBlendRecipe(Phi, Masks);
 }
 
-VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(Instruction *I,
-                                                   VFRange &Range) {
+VPWidenCallRecipe *
+VPRecipeBuilder::tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan) {
 
   bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
 
   CallInst *CI = dyn_cast<CallInst>(I);
   if (IsPredicated || !CI)
     return nullptr;
 
bool UseVectorIntrinsic =
         ID && CM.getVectorIntrinsicCost(CI, VF) <= CallCost;
     return UseVectorIntrinsic || !NeedToScalarize;
   };
 
   if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
     return nullptr;
 
   // Success: widen this call.
-  return new VPWidenCallRecipe(*CI);
+  auto VPValues = map_range(CI->arg_operands(), [&Plan](Value *Op) {
+    return Plan.getOrAddVPValue(Op);
+  });
+  SmallVector<VPValue *, 4> Values(VPValues.begin(), VPValues.end());
+
+  return new VPWidenCallRecipe(*CI, Values);
 }
 
 VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range,
                                            VPlan &Plan) {
   bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
 
   if (IsPredicated)

 }
 
 bool VPRecipeBuilder::tryToCreateRecipe(Instruction *Instr, VFRange &Range,
                                         VPlanPtr &Plan, VPBasicBlock *VPBB) {
   VPRecipeBase *Recipe = nullptr;
 
   // First, check for specific widening recipes that deal with calls, memory
   // operations, inductions and Phi nodes.
-  if ((Recipe = tryToWidenCall(Instr, Range)) ||
+  if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) ||
       (Recipe = tryToWidenMemory(Instr, Range, Plan)) ||
       (Recipe = tryToOptimizeInduction(Instr, Range)) ||
       (Recipe = tryToBlend(Instr, Plan)) ||
       (isa<PHINode>(Instr) &&
        (Recipe = new VPWidenPHIRecipe(cast<PHINode>(Instr))))) {
     setRecipe(Instr, Recipe);
     VPBB->appendRecipe(Recipe);
     return true;
void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
   O << "\\l\"";
   for (unsigned i = 0; i < IG->getFactor(); ++i)
     if (Instruction *I = IG->getMember(i))
       O << " +\n"
         << Indent << "\"  " << VPlanIngredient(I) << " " << i << "\\l\"";
 }
 
 void VPWidenCallRecipe::execute(VPTransformState &State) {
-  State.ILV->widenCallInstruction(Ingredient);
+  State.ILV->widenCallInstruction(Ingredient, User, State);
 }
 
 void VPWidenRecipe::execute(VPTransformState &State) {
   // Helpers to access the underlying value of a VPValue operand.
   auto GetUnderlyingValue = [this](unsigned OpIdx) {
     return User.getOperand(OpIdx)->getUnderlyingValue();
   };
 

llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

public:
   /// Handle non-loop phi nodes. Currently all such phi nodes are turned into
   /// a sequence of select instructions as the vectorizer currently performs
   /// full if-conversion.
   VPBlendRecipe *tryToBlend(Instruction *I, VPlanPtr &Plan);
 
   /// Handle call instruction. If \p I is a call that can be widened for \p
   /// Range.Start, return a new VPWidenCallRecipe. Range.End may be decreased to
   /// ensure same decision from \p Range.Start to \p Range.End.
-  VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range);
+  VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range,
+                                    VPlan &Plan);
 
   /// Check if \p I can be widened within the given VF \p Range. If \p I can be
   /// widened for \p Range.Start, build a new VPWidenRecipe and return it.
   /// Range.End may be decreased to ensure same decision from \p Range.Start to
   /// \p Range.End.
   VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range, VPlan &Plan);
 
   /// Create a replicating region for instruction \p I that requires

llvm/lib/Transforms/Vectorize/VPlan.h

 };
 
 /// A recipe for widening Call instructions.
 class VPWidenCallRecipe : public VPRecipeBase {
 private:
   /// Hold the call to be widened.
   CallInst &Ingredient;
 
+  /// Hold VPValues for the arguments of the call.
+  VPUser User;
+
 public:
-  VPWidenCallRecipe(CallInst &I) : VPRecipeBase(VPWidenCallSC), Ingredient(I) {}
+  VPWidenCallRecipe(CallInst &I, ArrayRef<VPValue *> CallArguments)
+      : VPRecipeBase(VPWidenCallSC), Ingredient(I), User(CallArguments) {}
 
   ~VPWidenCallRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPRecipeBase *V) {
     return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC;
   }