[VPlan] Make VPWidenMemoryInstructionRecipe a VPDef.

[fhahn]

Summary

This patch updates VPWidenMemoryInstructionRecipe to use VPDef
to manage the value it produces instead of inheriting from VPValue.

[fhahn]

Small changes to fix remaining test failures.

[fhahn]

Updated to keep VPWidenMemoryInstructionRecipe a VPValue which is then added to its VPDef. This simplifies things a bit.

[fhahn]

Update after VPDef does not inherit from VPUser directly.

[fhahn]

ping :)

The main patches have already been approved (D90565, D90564) and this patch just updates VPWidenMemoryInstructionRecipe to be a single-value def.

[fhahn]

ping

[gilr]

llvm/lib/Transforms/Vectorize/VPlan.h

1236

With this recipe being a VPDef we can drop the VPValue inheritance and create a VPValue (only) for loads, right?

[fhahn]

Thanks for taking a look Gil! Removed inheritance from VPValue, only create VPValue for loads.

llvm/lib/Transforms/Vectorize/VPlan.h

1236

That's a good point, we should be able to model this better now. Updated the patch

[gilr]

LGTM, tx!

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

if (State.ValueMap.hasVectorValue(PredInst, Part)) {
     PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
     Phi->addIncoming(UndefValue::get(ScalarPredInst->getType()), PredicatingBB);
     Phi->addIncoming(ScalarPredInst, PredicatedBB);
     State.ValueMap.resetScalarValue(PredInst, *State.Instance, Phi);
   }
 }
 
 void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
-  Instruction *Instr = getUnderlyingInstr();
-  VPValue *StoredValue = isa<StoreInst>(Instr) ? getStoredValue() : nullptr;
-  State.ILV->vectorizeMemoryInstruction(Instr, State,
-                                        StoredValue ? nullptr : this, getAddr(),
-                                        StoredValue, getMask());
+  VPValue *StoredValue = isStore() ? getStoredValue() : nullptr;
+  State.ILV->vectorizeMemoryInstruction(&Instr, State,
+                                        StoredValue ? nullptr : getVPValue(),
+                                        getAddr(), StoredValue, getMask());
 }
 
 // Determine how to lower the scalar epilogue, which depends on 1) optimising
 // for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
 // predication, and 4) a TTI hook that analyses whether the loop is suitable
 // for predication.
 static ScalarEpilogueLowering getScalarEpilogueLowering(
     Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI,

llvm/lib/Transforms/Vectorize/VPlan.h

 };
 
 /// A Recipe for widening load/store operations.
 /// The recipe uses the following VPValues:
 /// - For load: Address, optional mask
 /// - For store: Address, stored value, optional mask
 /// TODO: We currently execute only per-part unless a specific instance is
 /// provided.
-class VPWidenMemoryInstructionRecipe : public VPRecipeBase,
-                                       public VPValue,
-                                       public VPUser {
+class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPDef {
 
+  Instruction &Instr;
   void setMask(VPValue *Mask) {
     if (!Mask)
       return;
     addOperand(Mask);
   }
 
   bool isMasked() const {
-    return (isa<LoadInst>(getUnderlyingInstr()) && getNumOperands() == 2) ||
-           (isa<StoreInst>(getUnderlyingInstr()) && getNumOperands() == 3);
+    return isStore() ? getNumOperands() == 3 : getNumOperands() == 2;
   }
 
 public:
   VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask)
-      : VPRecipeBase(VPWidenMemoryInstructionSC),
-        VPValue(VPValue::VPMemoryInstructionSC, &Load), VPUser({Addr}) {
+      : VPRecipeBase(VPWidenMemoryInstructionSC), VPDef({Addr}), Instr(Load) {
+    new VPValue(&Load, this);
     setMask(Mask);
   }
 
   VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr,
                                  VPValue *StoredValue, VPValue *Mask)
-      : VPRecipeBase(VPWidenMemoryInstructionSC),
-        VPValue(VPValue::VPMemoryInstructionSC, &Store),
-        VPUser({Addr, StoredValue}) {
+      : VPRecipeBase(VPWidenMemoryInstructionSC), VPDef({Addr, StoredValue}),
+        Instr(Store) {
     setMask(Mask);
   }
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPRecipeBase *V) {
     return V->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC;
   }
 
   /// Return the address accessed by this recipe.
   VPValue *getAddr() const {
     return getOperand(0); // Address is the 1st, mandatory operand.
   }
 
   /// Return the mask used by this recipe. Note that a full mask is represented
   /// by a nullptr.
   VPValue *getMask() const {
     // Mask is optional and therefore the last operand.
     return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
   }
 
+  bool isStore() const { return getNumDefinedValues() == 0; }
+
   /// Return the address accessed by this recipe.
   VPValue *getStoredValue() const {
-    assert(isa<StoreInst>(getUnderlyingInstr()) &&
-           "Stored value only available for store instructions");
+    assert(isStore() && "Stored value only available for store instructions");
     return getOperand(1); // Stored value is the 2nd, mandatory operand.
   }
 
   /// Generate the wide load/store.
   void execute(VPTransformState &State) override;
 
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,

llvm/lib/Transforms/Vectorize/VPlan.cpp

if (auto *U = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
     return U;
   return nullptr;
 }
 
 VPValue *VPRecipeBase::toVPValue() {
   if (auto *V = dyn_cast<VPInstruction>(this))
     return V;
   if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
-    return V;
+    if (V->getNumDefinedValues() == 1)
+      return V->getVPValue();
   if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
     return V->getVPValue();
   if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
     return V->getVPValue();
   if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
     return V->getVPValue();
   return nullptr;
 }
 
 const VPValue *VPRecipeBase::toVPValue() const {
   if (auto *V = dyn_cast<VPInstruction>(this))
     return V;
   if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
-    return V;
+    if (V->getNumDefinedValues() == 1)
+      return V->getVPValue();
   if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
     return V->getVPValue();
   if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
     return V->getVPValue();
   if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
     return V->getVPValue();
   return nullptr;
 }

 
 void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                                 VPSlotTracker &SlotTracker) const {
   O << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst);
 }
 
 void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent,
                                            VPSlotTracker &SlotTracker) const {
-  O << "\"WIDEN "
-    << Instruction::getOpcodeName(getUnderlyingInstr()->getOpcode()) << " ";
+  O << "\"WIDEN " << Instruction::getOpcodeName(Instr.getOpcode()) << " ";
 
   bool First = true;
   for (VPValue *Op : operands()) {
     if (!First)
       O << ", ";
     Op->printAsOperand(O, SlotTracker);
     First = false;
   }