[VPlan] Add VPReplicateRecipe::Instr field to hold replicated instr.

[fhahn]

Summary

At the moment, VPReplicateRecipe uses the underlying value to store the
replicated instruction. This makes it a bit awkward to introduce new
VPReplicateRecipes or clone existing ones.

Instead, add a dedicated pointer to hold the instruction, but make it
clear that the operands and uses are modeled in VPlan; the instruction
is only used for opcode, type info, metadata and so on.

[Ayal]

Underlying value is designed to only be used for opcode, type info, metadata and so on, for any recipe; the operands and uses are modeled in VPlan, for any recipe; worth clarifying?

Perhaps Replicate recipe should support cloning or copy construction?

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

case Intrinsic::lifetime_end:
       //      remove the call.
       IsUniform = true;
       break;
     default:
       break;
     }
   }
 
-  auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()),
+  auto *Recipe = new VPReplicateRecipe(I, I, Plan->mapToVPValues(I->operands()),
                                        IsUniform, IsPredicated);
 
   // Find if I uses a predicated instruction. If so, it will use its scalar
   // value. Avoid hoisting the insert-element which packs the scalar value into
   // a vector value, as that happens iff all users use the vector value.
   for (VPValue *Op : Recipe->operands()) {
     auto *PredR = dyn_cast_or_null<VPPredInstPHIRecipe>(Op->getDef());
     if (!PredR)
else
       NextInChain = State.Builder.CreateBinOp(
           (Instruction::BinaryOps)RdxDesc->getOpcode(Kind), NewRed,
           PrevInChain);
     State.set(this, NextInChain, Part);
   }
 }
 
 void VPReplicateRecipe::execute(VPTransformState &State) {
-  Instruction *UI = getUnderlyingInstr();
   if (State.Instance) { // Generate a single instance.
     assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
-    State.ILV->scalarizeInstruction(UI, this, *State.Instance,
-                                    IsPredicated, State);
+    State.ILV->scalarizeInstruction(Instr, this, *State.Instance, IsPredicated,
+                                    State);
     // Insert scalar instance packing it into a vector.
     if (AlsoPack && State.VF.isVector()) {
       // If we're constructing lane 0, initialize to start from poison.
       if (State.Instance->Lane.isFirstLane()) {
         assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
-        Value *Poison = PoisonValue::get(
-            VectorType::get(UI->getType(), State.VF));
+        Value *Poison =
+            PoisonValue::get(VectorType::get(Instr->getType(), State.VF));
         State.set(this, Poison, State.Instance->Part);
       }
       State.ILV->packScalarIntoVectorValue(this, *State.Instance, State);
     }
     return;
   }
 
   if (IsUniform) {
     // If the recipe is uniform across all parts (instead of just per VF), only
     // generate a single instance.
-    if ((!mayHaveSideEffects() || isa<StoreInst>(UI)) &&
-        all_of(operands(), [](VPValue *Op) { return Op->isDefinedOutsideVectorRegions(); })) {
-      State.ILV->scalarizeInstruction(UI, this, VPIteration(0, 0), IsPredicated,
-                                      State);
+    if ((!mayHaveSideEffects() || isa<StoreInst>(Instr)) &&
+        all_of(operands(), [](VPValue *Op) {
+          return Op->isDefinedOutsideVectorRegions();
+        })) {
+      State.ILV->scalarizeInstruction(Instr, this, VPIteration(0, 0),
+                                      IsPredicated, State);
       if (user_begin() != user_end()) {
         for (unsigned Part = 1; Part < State.UF; ++Part)
           State.set(this, State.get(this, VPIteration(0, 0)),
                     VPIteration(Part, 0));
       }
       return;
     }
 
     // Uniform within VL means we need to generate lane 0 only for each
     // unrolled copy.
     for (unsigned Part = 0; Part < State.UF; ++Part)
-      State.ILV->scalarizeInstruction(UI, this, VPIteration(Part, 0),
+      State.ILV->scalarizeInstruction(Instr, this, VPIteration(Part, 0),
                                       IsPredicated, State);
     return;
   }
 
   // A store of a loop varying value to a loop invariant address only
   // needs a single copy of the store.
-  if (isa<StoreInst>(UI) && !getOperand(1)->getDef()) {
+  if (isa<StoreInst>(Instr) && !getOperand(1)->getDef()) {
     auto Lane = VPLane::getLastLaneForVF(State.VF);
-    State.ILV->scalarizeInstruction(UI, this, VPIteration(State.UF - 1, Lane), IsPredicated,
-                                    State);
+    State.ILV->scalarizeInstruction(
+        Instr, this, VPIteration(State.UF - 1, Lane), IsPredicated, State);
     return;
   }
 
   // Generate scalar instances for all VF lanes of all UF parts.
   assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
   const unsigned EndLane = State.VF.getKnownMinValue();
   for (unsigned Part = 0; Part < State.UF; ++Part)
     for (unsigned Lane = 0; Lane < EndLane; ++Lane)
-      State.ILV->scalarizeInstruction(UI, this, VPIteration(Part, Lane),
+      State.ILV->scalarizeInstruction(Instr, this, VPIteration(Part, Lane),
                                       IsPredicated, State);
 }
 
 void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
   VPValue *StoredValue = isStore() ? getStoredValue() : nullptr;
 
   // Attempt to issue a wide load.
   LoadInst *LI = dyn_cast<LoadInst>(&Ingredient);

llvm/lib/Transforms/Vectorize/VPlan.h

#endif
   }
 };
 
 /// VPReplicateRecipe replicates a given instruction producing multiple scalar
 /// copies of the original scalar type, one per lane, instead of producing a
 /// single copy of widened type for all lanes. If the instruction is known to be
 /// uniform only one copy, per lane zero, will be generated.
 class VPReplicateRecipe : public VPRecipeBase, public VPValue {
+  /// The instruction being replicated. It is only used for opcode, type info,
+  /// attributes and metadata. Operands and uses of the result are modeled in
+  /// VPlan.
+  const Instruction *Instr;
+
   /// Indicator if only a single replica per lane is needed.
   bool IsUniform;
 
   /// Indicator if the replicas are also predicated.
   bool IsPredicated;
 
   /// Indicator if the scalar values should also be packed into a vector.
   bool AlsoPack;
 
 public:
   template <typename IterT>
-  VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
-                    bool IsUniform, bool IsPredicated = false)
-      : VPRecipeBase(VPReplicateSC, Operands), VPValue(VPVReplicateSC, I, this),
+  VPReplicateRecipe(const Instruction *I, Value *UnderlyingValue,
+                    iterator_range<IterT> Operands, bool IsUniform,
+                    bool IsPredicated = false)
+      : VPRecipeBase(VPReplicateSC, Operands),
+        VPValue(VPVReplicateSC, UnderlyingValue, this), Instr(I),
         IsUniform(IsUniform), IsPredicated(IsPredicated) {
     // Retain the previous behavior of predicateInstructions(), where an
     // insert-element of a predicated instruction got hoisted into the
     // predicated basic block iff it was its only user. This is achieved by
     // having predicated instructions also pack their values into a vector by
     // default unless they have a replicated user which uses their scalar value.
     AlsoPack = IsPredicated && !I->use_empty();
   }
#endif
   }
 
   /// Returns true if the recipe uses scalars of operand \p Op.
   bool usesScalars(const VPValue *Op) const override {
     assert(is_contained(operands(), Op) &&
            "Op must be an operand of the recipe");
     return true;
   }
+
+  const Instruction &getInstruction() const { return *Instr; }
 };
 
 /// A recipe for generating conditional branches on the bits of a mask.
 class VPBranchOnMaskRecipe : public VPRecipeBase {
 public:
   VPBranchOnMaskRecipe(VPValue *BlockInMask)
       : VPRecipeBase(VPBranchOnMaskSC, {}) {
     if (BlockInMask) // nullptr means all-one mask.

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

 #define LV_NAME "loop-vectorize"
 #define DEBUG_TYPE LV_NAME
 
 bool VPRecipeBase::mayWriteToMemory() const {
   switch (getVPDefID()) {
   case VPWidenMemoryInstructionSC: {
     return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
   }
-  case VPReplicateSC:
+  case VPReplicateSC: {
+    auto *R = cast<VPReplicateRecipe>(this);
+    return R->getInstruction().mayWriteToMemory();
+  }
   case VPWidenCallSC:
     return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
         ->mayWriteToMemory();
   case VPBranchOnMaskSC:
     return false;
   case VPWidenIntOrFpInductionSC:
   case VPWidenCanonicalIVSC:
   case VPWidenPHISC:
bool VPRecipeBase::mayWriteToMemory() const {
   }
 }
 
 bool VPRecipeBase::mayReadFromMemory() const {
   switch (getVPDefID()) {
   case VPWidenMemoryInstructionSC: {
     return !cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
   }
-  case VPReplicateSC:
+  case VPReplicateSC: {
+    auto *R = cast<VPReplicateRecipe>(this);
+    return R->getInstruction().mayReadFromMemory();
+  }
   case VPWidenCallSC:
     return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
         ->mayReadFromMemory();
   case VPBranchOnMaskSC:
     return false;
   case VPWidenIntOrFpInductionSC:
   case VPWidenCanonicalIVSC:
   case VPWidenPHISC:
const Instruction *I =
         dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
     (void)I;
     assert((!I || !I->mayHaveSideEffects()) &&
            "underlying instruction has side-effects");
     return false;
   }
   case VPReplicateSC: {
     auto *R = cast<VPReplicateRecipe>(this);
-    return R->getUnderlyingInstr()->mayHaveSideEffects();
+    return R->getInstruction().mayHaveSideEffects();
   }
   default:
     return true;
   }
 }
 
 void VPLiveOut::fixPhi(VPlan &Plan, VPTransformState &State) {
   auto Lane = VPLane::getLastLaneForVF(State.VF);
if (RdxDesc->IntermediateStore)
     O << " (with final reduction value stored in invariant address sank "
          "outside of loop)";
 }
 
 void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent,
                               VPSlotTracker &SlotTracker) const {
   O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
 
-  if (!getUnderlyingInstr()->getType()->isVoidTy()) {
+  if (!Instr->getType()->isVoidTy()) {
     printAsOperand(O, SlotTracker);
     O << " = ";
   }
-  if (auto *CB = dyn_cast<CallBase>(getUnderlyingInstr())) {
+  if (auto *CB = dyn_cast<CallBase>(Instr)) {
     O << "call @" << CB->getCalledFunction()->getName() << "(";
     interleaveComma(make_range(op_begin(), op_begin() + (getNumOperands() - 1)),
                     O, [&O, &SlotTracker](VPValue *Op) {
                       Op->printAsOperand(O, SlotTracker);
                     });
     O << ")";
   } else {
-    O << Instruction::getOpcodeName(getUnderlyingInstr()->getOpcode()) << " ";
+    O << Instruction::getOpcodeName(Instr->getOpcode()) << " ";
     printOperands(O, SlotTracker);
   }
 
   if (AlsoPack)
     O << " (S->V)";
 }
 #endif
 

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

auto CanSinkWithUser = [SinkTo, &NeedsDuplicating, C](VPUser *U) {
         return true;
       }
       return false;
     };
     if (!all_of(C->users(), CanSinkWithUser))
       continue;
 
     if (NeedsDuplicating) {
-      Instruction *I = cast<Instruction>(
-          cast<VPReplicateRecipe>(SinkCandidate)->getUnderlyingValue());
-      auto *Clone =
-          new VPReplicateRecipe(I, SinkCandidate->operands(), true, false);
+      const Instruction *I =
+          &cast<VPReplicateRecipe>(SinkCandidate)->getInstruction();
+      auto *Clone = new VPReplicateRecipe(I, nullptr, SinkCandidate->operands(),
+                                          true, false);
       // TODO: add ".cloned" suffix to name of Clone's VPValue.
 
       Clone->insertBefore(SinkCandidate);
       SmallVector<VPUser *, 4> Users(C->users());
       for (auto *U : Users) {
         auto *UI = cast<VPRecipeBase>(U);
         if (UI->getParent() == SinkTo)
           continue;

llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll

 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>
-; CHECK-NEXT:   CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT:   CLONE vp<[[CLONED_GEP:%.+]]> = getelementptr ir<%addr>, vp<[[STEPS]]>
 ; CHECK-NEXT: Successor(s): loop.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.body:
-; CHECK-NEXT:   WIDEN ir<%0> = load ir<%gep>
+; CHECK-NEXT:   WIDEN ir<%0> = load vp<[[CLONED_GEP]]>
 ; CHECK-NEXT:   WIDEN ir<%pred> = fcmp ir<%0>, ir<0.000000e+00>
 ; CHECK-NEXT: Successor(s): then
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then:
 ; CHECK-NEXT:   EMIT vp<[[MASK:%.+]]> = not ir<%pred>
 ; CHECK-NEXT: Successor(s): pred.store
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.store: {

llvm/unittests/Transforms/Vectorize/VPlanTest.cpp

TEST(VPRecipeTest, CastVPReplicateRecipeToVPUser) {
   LLVMContext C;
 
   VPValue Op1;
   VPValue Op2;
   SmallVector<VPValue *, 4> Args;
   Args.push_back(&Op1);
   Args.push_back(&Op2);
 
-  VPReplicateRecipe Recipe(nullptr, make_range(Args.begin(), Args.end()), true,
-                           false);
+  VPReplicateRecipe Recipe(nullptr, nullptr,
+                           make_range(Args.begin(), Args.end()), true, false);
   EXPECT_TRUE(isa<VPUser>(&Recipe));
   VPRecipeBase *BaseR = &Recipe;
   EXPECT_TRUE(isa<VPUser>(BaseR));
 }
 
 TEST(VPRecipeTest, CastVPBranchOnMaskRecipeToVPUser) {
   LLVMContext C;