[VPlan] Use VPValue def for VPWidenCall.

[fhahn]

Summary

This patch turns VPWidenCall into a VPValue and uses it
during VPlan construction and codegeneration instead of the plain IR
reference where possible.

[dmgreen]

llvm/lib/Transforms/Vectorize/VPlan.h

857

VPUser too? Or is that simpler to do later?

861–862

VPRecipeBase(VPWidenCallSC) -> VPRecipeBase(VPRecipeBase::VPWidenCallSC). To make it explicit whilst there are two.

[fhahn]

rebase

[fhahn]

Addressed @dmgreen's comment, thanks! Also updated code to just use shared VPRecipeBase::getUnderlyingInstr()

llvm/lib/Transforms/Vectorize/VPlan.h

861–862

done, thanks

[dmgreen]

Looks straight forward enough.

[fhahn]

Change VPDef to not inherit from VPUser. This gives a bit more flexibility and VDef can probably be folded into VPRecipeBase after the transition is complete.

[fhahn]

Rebased and fixed printing. I plan to commit this shortly, as all approaches discussed will require updating codegen & co to operate on VPValues, so those patches should be needed in any case.

[fhahn]

rebase again

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

public:
   /// loop.
   BasicBlock *createVectorizedLoopSkeleton();
 
   /// Widen a single instruction within the innermost loop.
   void widenInstruction(Instruction &I, VPUser &Operands,
                         VPTransformState &State);
 
   /// Widen a single call instruction within the innermost loop.
-  void widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+  void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands,
                             VPTransformState &State);
 
   /// Widen a single select instruction within the innermost loop.
   void widenSelectInstruction(SelectInst &I, VPUser &Operands,
                               bool InvariantCond, VPTransformState &State);
 
   /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
   void fixVectorizedLoop();
void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User,
   }
   default:
     // This instruction is not vectorized by simple widening.
     LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
     llvm_unreachable("Unhandled instruction!");
   } // end of switch.
 }
 
-void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
+                                               VPUser &ArgOperands,
                                                VPTransformState &State) {
   assert(!isa<DbgInfoIntrinsic>(I) &&
          "DbgInfoIntrinsic should have been dropped during VPlan construction");
   setDebugLocFromInst(Builder, &I);
 
   Module *M = I.getParent()->getParent()->getParent();
   auto *CI = cast<CallInst>(&I);
 
#endif
     }
       SmallVector<OperandBundleDef, 1> OpBundles;
       CI->getOperandBundlesAsDefs(OpBundles);
       CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
 
       if (isa<FPMathOperator>(V))
         V->copyFastMathFlags(CI);
 
-      VectorLoopValueMap.setVectorValue(&I, Part, V);
+      State.set(Def, &I, V, Part);
       addMetadata(V, &I);
   }
 }
 
 void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I,
                                                  VPUser &Operands,
                                                  bool InvariantCond,
                                                  VPTransformState &State) {
if (Mask) {
     Mask->printAsOperand(O, SlotTracker);
   }
   for (unsigned i = 0; i < IG->getFactor(); ++i)
     if (Instruction *I = IG->getMember(i))
       O << "\\l\" +\n" << Indent << "\"  " << VPlanIngredient(I) << " " << i;
 }
 
 void VPWidenCallRecipe::execute(VPTransformState &State) {
-  State.ILV->widenCallInstruction(Ingredient, *this, State);
+  State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
+                                  *this, State);
 }
 
 void VPWidenSelectRecipe::execute(VPTransformState &State) {
   State.ILV->widenSelectInstruction(Ingredient, *this, InvariantCond, State);
 }
 
 void VPWidenRecipe::execute(VPTransformState &State) {
   State.ILV->widenInstruction(Ingredient, *this, State);

llvm/lib/Transforms/Vectorize/VPlan.h

public:
   void execute(VPTransformState &State) override;
 
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
 };
 
 /// A recipe for widening Call instructions.
-class VPWidenCallRecipe : public VPRecipeBase, public VPUser {
-  /// Hold the call to be widened.
-  CallInst &Ingredient;
+class VPWidenCallRecipe : public VPRecipeBase, public VPValue, public VPUser {
 
 public:
   template <typename IterT>
   VPWidenCallRecipe(CallInst &I, iterator_range<IterT> CallArguments)
-      : VPRecipeBase(VPWidenCallSC), VPUser(CallArguments), Ingredient(I) {}
+      : VPRecipeBase(VPRecipeBase::VPWidenCallSC),
+        VPValue(VPValue::VPVWidenCallSC, &I), VPUser(CallArguments) {}
 
   ~VPWidenCallRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPRecipeBase *V) {
     return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC;
   }
 

llvm/lib/Transforms/Vectorize/VPlan.cpp

VPUser *VPRecipeBase::toVPUser() {
   return nullptr;
 }
 
 VPValue *VPRecipeBase::toVPValue() {
   if (auto *V = dyn_cast<VPInstruction>(this))
     return V;
   if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
     return V;
+  if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
+    return V;
   return nullptr;
 }
 
 const VPValue *VPRecipeBase::toVPValue() const {
   if (auto *V = dyn_cast<VPInstruction>(this))
     return V;
   if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
     return V;
+  if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
+    return V;
   return nullptr;
 }
 
 // Get the top-most entry block of \p Start. This is the entry block of the
 // containing VPlan. This function is templated to support both const and non-const blocks
 template <typename T> static T *getPlanEntry(T *Start) {
   T *Next = Start;
   T *Current = Start;
void VPlanPrinter::printAsIngredient(raw_ostream &O, const Value *V) {
   } else // !Inst
     V->printAsOperand(RSO, false);
   RSO.flush();
   O << DOT::EscapeString(IngredientString);
 }
 
 void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
                               VPSlotTracker &SlotTracker) const {
-  O << "\"WIDEN-CALL " << VPlanIngredient(&Ingredient);
+  O << "\"WIDEN-CALL ";
+
+  auto *CI = cast<CallInst>(getUnderlyingInstr());
+  if (CI->getType()->isVoidTy())
+    O << "void ";
+  else {
+    printAsOperand(O, SlotTracker);
+    O << " = ";
+  }
+
+  O << "call @" << CI->getCalledFunction()->getName() << "(";
+  printOperands(O, SlotTracker);
+  O << ")";
 }
 
 void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent,
                                 VPSlotTracker &SlotTracker) const {
   O << "\"WIDEN-SELECT" << VPlanIngredient(&Ingredient)
     << (InvariantCond ? " (condition is loop invariant)" : "");
 }
 

llvm/lib/Transforms/Vectorize/VPlanValue.h

void setUnderlyingValue(Value *Val) {
     UnderlyingVal = Val;
   }
 
 public:
   /// An enumeration for keeping track of the concrete subclass of VPValue that
   /// are actually instantiated. Values of this enumeration are kept in the
   /// SubclassID field of the VPValue objects. They are used for concrete
   /// type identification.
-  enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC };
+  enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC };
 
   VPValue(Value *UV = nullptr, VPDef *Def = nullptr)
       : VPValue(VPValueSC, UV, Def) {}
   VPValue(const VPValue &) = delete;
   VPValue &operator=(const VPValue &) = delete;
 
   virtual ~VPValue() {
     assert(Users.empty() && "trying to delete a VPValue with remaining users");

llvm/test/Transforms/LoopVectorize/vplan-printing.ll

+; REQUIRES: asserts
+
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -disable-output %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Tests for printing VPlans.
+
+define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+; CHECK: N0 [label =
+; CHECK-NEXT: "for.body:\n" +
+; CHECK-NEXT:       "WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
+; CHECK-NEXT:       "CLONE %arrayidx = getelementptr %y, %iv\l" +
+; CHECK-NEXT:       "WIDEN load ir<%arrayidx>\l" +
+; CHECK-NEXT:       "WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)\l" +
+; CHECK-NEXT:       "CLONE %arrayidx2 = getelementptr %x, %iv\l" +
+; CHECK-NEXT:       "WIDEN store ir<%arrayidx2>, ir<%call>\l"
+; CHECK-NEXT:   ]
+
+entry:
+  %cmp6 = icmp sgt i64 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %iv
+  %lv = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.sqrt.f32(float %lv) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %iv
+  store float %call, float* %arrayidx2, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.sqrt.f32(float) nounwind readnone