[VPlan] Make VPInstruction a VPDef

[fhahn]

Summary

This patch turns updates VPInstruction to manage the value it defines
using VPDef. The VPValue is used during VPlan construction and
codegeneration instead of the plain IR reference where possible.

[fhahn]

Completely remove toVPValue

[fhahn]

Updated to keep VPWidenMemoryInstructionRecipe a VPValue which is then added to its VPDef. This simplifies things a lot here.

[fhahn]

Update after VPDef does not inherit from VPUser directly.

[fhahn]

Another round of re-basing

[gilr]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

7802

Is this still possible? Shouldn't e.g. WidenMemoryInstruction, InterleaveGroup, WidenCall recipes hold an empty VPDef for stores / functions returning void?

[fhahn]

Remove obsolete isVoidTy check.

[fhahn]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

7802

Good catch, with the VPDef approach, checking for VoidTy is not needed any longer. Removed!

[fhahn]

Remove now obsolete toVPValue.

[gilr]

LGTM, tx!

[fhahn]

rebase after parent changes landed.

llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

VPInstruction *insert(VPInstruction *I) const {
     return I;
   }
 
   /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
   /// its underlying Instruction.
   VPValue *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
                         Instruction *Inst = nullptr) {
     VPInstruction *NewVPInst = createInstruction(Opcode, Operands);
-    NewVPInst->setUnderlyingValue(Inst);
-    return NewVPInst;
+    NewVPInst->setUnderlyingInstr(Inst);
+    return NewVPInst->getVPValue();
   }
   VPValue *createNaryOp(unsigned Opcode,
                         std::initializer_list<VPValue *> Operands,
                         Instruction *Inst = nullptr) {
     return createNaryOp(Opcode, ArrayRef<VPValue *>(Operands), Inst);
   }
 
   VPValue *createNot(VPValue *Operand) {
-    return createInstruction(VPInstruction::Not, {Operand});
+    return createInstruction(VPInstruction::Not, {Operand})->getVPValue();
   }
 
   VPValue *createAnd(VPValue *LHS, VPValue *RHS) {
-    return createInstruction(Instruction::BinaryOps::And, {LHS, RHS});
+    return createInstruction(Instruction::BinaryOps::And, {LHS, RHS})
+        ->getVPValue();
   }
 
   VPValue *createOr(VPValue *LHS, VPValue *RHS) {
-    return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS});
+    return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS})
+        ->getVPValue();
   }
 
   //===--------------------------------------------------------------------===//
   // RAII helpers.
   //===--------------------------------------------------------------------===//
 
   /// RAII object that stores the current insertion point and restores it when
   /// the object is destroyed.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

for (Instruction &I : BB->instructionsWithoutDebug()) {
       // First filter out irrelevant instructions, to ensure no recipes are
       // built for them.
       if (isa<BranchInst>(Instr) || DeadInstructions.count(Instr) ||
           DeadInterleaveGroupMembers.contains(Instr))
         continue;
 
       if (auto Recipe =
               RecipeBuilder.tryToCreateWidenRecipe(Instr, Range, Plan)) {
-        // Check if the recipe can be converted to a VPValue. We need the extra
-        // down-casting step until VPRecipeBase inherits from VPValue.
-        VPValue *MaybeVPValue = Recipe->toVPValue();
-        if (!Instr->getType()->isVoidTy() && MaybeVPValue) {
-          if (NeedDef.contains(Instr))
-            Plan->addOrReplaceVPValue(Instr, MaybeVPValue);
+        auto RegisterVPValue = [&Plan, &NeedDef](Value *V, VPValue *Def) {
+          if (NeedDef.contains(V))
+            Plan->addOrReplaceVPValue(V, Def);
           else
-            Plan->addVPValue(Instr, MaybeVPValue);
+            Plan->addVPValue(V, Def);
+        };
+
+        if (Recipe->getNumDefinedValues() > 1) {
+          for (auto *Def : Recipe->defined_values()) {
+            auto *UV = Def->getUnderlyingValue();
+            RegisterVPValue(UV, Def);
+          }
+        } else if (Recipe->getNumDefinedValues() != 0) {
+          assert(Recipe->getNumDefinedValues() == 1 &&
+                 "all multi-defs should be handled earlier");
+          RegisterVPValue(&I, Recipe->getVPValue());
         }
 
         RecipeBuilder.setRecipe(Instr, Recipe);
         VPBB->appendRecipe(Recipe);
         continue;
       }
 
       // Otherwise, if all widening options failed, Instruction is to be

llvm/lib/Transforms/Vectorize/VPlan.h

public:
   /// delete it.
   void removeFromParent();
 
   /// This method unlinks 'this' from the containing basic block and deletes it.
   ///
   /// \returns an iterator pointing to the element after the erased one
   iplist<VPRecipeBase>::iterator eraseFromParent();
 
-  /// Returns a pointer to a VPValue, if the recipe inherits from VPValue or
-  /// nullptr otherwise.
-  VPValue *toVPValue();
-  const VPValue *toVPValue() const;
-
   /// Returns the underlying instruction, if the recipe is a VPValue or nullptr
   /// otherwise.
   Instruction *getUnderlyingInstr() {
     if (getNumDefinedValues() == 1)
       return cast_or_null<Instruction>(getVPValue()->getUnderlyingValue());
     return nullptr;
   }
   const Instruction *getUnderlyingInstr() const {
return Recipe->getVPDefID() == VPRecipeBase::VPInstructionSC ||
          Recipe->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
          Recipe->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
 }
 
 /// This is a concrete Recipe that models a single VPlan-level instruction.
 /// While as any Recipe it may generate a sequence of IR instructions when
 /// executed, these instructions would always form a single-def expression as
 /// the VPInstruction is also a single def-use vertex.
-class VPInstruction : public VPValue, public VPRecipeBase {
+class VPInstruction : public VPRecipeBase {
   friend class VPlanSlp;
 
 public:
   /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
   enum {
     Not = Instruction::OtherOpsEnd + 1,
     ICmpULE,
     SLPLoad,
     SLPStore,
     ActiveLaneMask,
   };
 
 private:
   typedef unsigned char OpcodeTy;
   OpcodeTy Opcode;
 
   /// Utility method serving execute(): generates a single instance of the
   /// modeled instruction.
   void generateInstruction(VPTransformState &State, unsigned Part);
 
 protected:
-  void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); }
 
 public:
+  void setUnderlyingInstr(Instruction *I) {
+    getVPValue()->setUnderlyingValue(I);
+  }
   VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands)
-      : VPValue(VPValue::VPInstructionSC),
-        VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands), Opcode(Opcode) {}
+      : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands), Opcode(Opcode) {
+    new VPValue(nullptr, this);
+  }
 
   VPInstruction(unsigned Opcode, ArrayRef<VPInstruction *> Operands)
-      : VPValue(VPValue::VPInstructionSC),
-        VPRecipeBase(VPRecipeBase::VPInstructionSC, {}), Opcode(Opcode) {
+      : VPRecipeBase(VPRecipeBase::VPInstructionSC, {}), Opcode(Opcode) {
+    new VPValue(nullptr, this);
     for (auto *I : Operands)
       addOperand(I->getVPValue());
   }
 
   VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
       : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {}
 
-  /// Method to support type inquiry through isa, cast, and dyn_cast.
-  static inline bool classof(const VPValue *V) {
-    return V->getVPValueID() == VPValue::VPInstructionSC;
-  }
-
   VPInstruction *clone() const {
     SmallVector<VPValue *, 2> Operands(operands());
     return new VPInstruction(Opcode, Operands);
   }
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPDef *R) {
     return R->getVPDefID() == VPRecipeBase::VPInstructionSC;

llvm/lib/Transforms/Vectorize/VPlan.cpp

 #include <vector>
 
 using namespace llvm;
 extern cl::opt<bool> EnableVPlanNativePath;
 
 #define DEBUG_TYPE "vplan"
 
 raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
-  const VPInstruction *Instr = dyn_cast<VPInstruction>(&V);
+  const VPInstruction *Instr = dyn_cast<VPInstruction>(V.getDef());
   VPSlotTracker SlotTracker(
       (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
   V.print(OS, SlotTracker);
   return OS;
 }
 
-VPValue::VPValue(const unsigned char SC, Value *UV, VPDef *Def)
-    : SubclassID(SC), UnderlyingVal(UV), Def(Def) {
+VPValue::VPValue(Value *UV, VPDef *Def) : UnderlyingVal(UV), Def(Def) {
   if (Def)
     Def->addDefinedValue(this);
 }
 
 void VPValue::print(raw_ostream &OS, VPSlotTracker &SlotTracker) const {
-  if (const VPInstruction *Instr = dyn_cast<VPInstruction>(this))
+  if (const VPInstruction *Instr = dyn_cast_or_null<VPInstruction>(getDef()))
     Instr->print(OS, SlotTracker);
   else
     printAsOperand(OS, SlotTracker);
 }
 
 void VPValue::dump() const {
-  const VPInstruction *Instr = dyn_cast<VPInstruction>(this);
+  const VPInstruction *Instr = dyn_cast_or_null<VPInstruction>(getDef());
   VPSlotTracker SlotTracker(
       (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
   print(dbgs(), SlotTracker);
   dbgs() << "\n";
 }
 
 void VPRecipeBase::dump() const {
   VPSlotTracker SlotTracker(nullptr);
   print(dbgs(), "", SlotTracker);
   dbgs() << "\n";
 }
 
-VPValue *VPRecipeBase::toVPValue() {
-  if (auto *V = dyn_cast<VPInstruction>(this))
-    return V;
-  if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
-    if (V->getNumDefinedValues() == 1)
-      return V->getVPValue();
-  if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
-    return V->getVPValue();
-  if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
-    return V->getVPValue();
-  if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
-    return V->getVPValue();
-  return nullptr;
-}
-
-const VPValue *VPRecipeBase::toVPValue() const {
-  if (auto *V = dyn_cast<VPInstruction>(this))
-    return V;
-  if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
-    if (V->getNumDefinedValues() == 1)
-      return V->getVPValue();
-  if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
-    return V->getVPValue();
-  if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
-    return V->getVPValue();
-  if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
-    return V->getVPValue();
-  return nullptr;
-}
 
 // Get the top-most entry block of \p Start. This is the entry block of the
 // containing VPlan. This function is templated to support both const and non-const blocks
 template <typename T> static T *getPlanEntry(T *Start) {
   T *Next = Start;
   T *Current = Start;
   while ((Next = Next->getParent()))
     Current = Next;
if (EnableVPlanNativePath && (CBV = getCondBit())) {
     ReplaceInstWithInst(CurrentTerminator, CondBr);
   }
 
   LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB);
 }
 
 void VPBasicBlock::dropAllReferences(VPValue *NewValue) {
   for (VPRecipeBase &R : Recipes) {
-    if (VPValue *Def = R.toVPValue())
-      Def->replaceAllUsesWith(NewValue);
-    else if (auto *IR = dyn_cast<VPInterleaveRecipe>(&R)) {
-      for (auto *Def : IR->defined_values())
-        Def->replaceAllUsesWith(NewValue);
-    }
+    for (auto *Def : R.defined_values())
 
     for (unsigned I = 0, E = R.getNumOperands(); I != E; I++)
       R.setOperand(I, NewValue);
   }
 }
 
 void VPRegionBlock::execute(VPTransformState *State) {
   ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry);

 void VPInstruction::generateInstruction(VPTransformState &State,
                                         unsigned Part) {
   IRBuilder<> &Builder = State.Builder;
 
   if (Instruction::isBinaryOp(getOpcode())) {
     Value *A = State.get(getOperand(0), Part);
     Value *B = State.get(getOperand(1), Part);
     Value *V = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B);
-    State.set(this, V, Part);
+    State.set(getVPValue(), V, Part);
     return;
   }
 
   switch (getOpcode()) {
   case VPInstruction::Not: {
     Value *A = State.get(getOperand(0), Part);
     Value *V = Builder.CreateNot(A);
-    State.set(this, V, Part);
+    State.set(getVPValue(), V, Part);
     break;
   }
   case VPInstruction::ICmpULE: {
     Value *IV = State.get(getOperand(0), Part);
     Value *TC = State.get(getOperand(1), Part);
     Value *V = Builder.CreateICmpULE(IV, TC);
-    State.set(this, V, Part);
+    State.set(getVPValue(), V, Part);
     break;
   }
   case Instruction::Select: {
     Value *Cond = State.get(getOperand(0), Part);
     Value *Op1 = State.get(getOperand(1), Part);
     Value *Op2 = State.get(getOperand(2), Part);
     Value *V = Builder.CreateSelect(Cond, Op1, Op2);
-    State.set(this, V, Part);
+    State.set(getVPValue(), V, Part);
     break;
   }
   case VPInstruction::ActiveLaneMask: {
     // Get first lane of vector induction variable.
     Value *VIVElem0 = State.get(getOperand(0), {Part, 0});
     // Get the original loop tripcount.
     Value *ScalarTC = State.TripCount;
 
     auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
     auto *PredTy = FixedVectorType::get(Int1Ty, State.VF.getKnownMinValue());
     Instruction *Call = Builder.CreateIntrinsic(
         Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()},
         {VIVElem0, ScalarTC}, nullptr, "active.lane.mask");
-    State.set(this, Call, Part);
+    State.set(getVPValue(), Call, Part);
     break;
   }
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
 }
 
 void VPInstruction::execute(VPTransformState &State) {

 
 void VPInstruction::print(raw_ostream &O) const {
   VPSlotTracker SlotTracker(getParent()->getPlan());
   print(O, SlotTracker);
 }
 
 void VPInstruction::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
   if (hasResult()) {
-    printAsOperand(O, SlotTracker);
+    getVPValue()->printAsOperand(O, SlotTracker);
     O << " = ";
   }
 
   switch (getOpcode()) {
   case VPInstruction::Not:
     O << "not";
     break;
   case VPInstruction::ICmpULE:
void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
   bumpIndent(1);
   OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
   bumpIndent(1);
 
   // Dump the block predicate.
   const VPValue *Pred = BasicBlock->getPredicate();
   if (Pred) {
     OS << " +\n" << Indent << " \"BlockPredicate: ";
-    if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) {
-      PredI->printAsOperand(OS, SlotTracker);
+    if (const VPInstruction *PredI = Pred->getAs<VPInstruction>()) {
+      PredI->getVPValue()->printAsOperand(OS, SlotTracker);
       OS << " (" << DOT::EscapeString(PredI->getParent()->getName())
          << ")\\l\"";
     } else
       Pred->printAsOperand(OS, SlotTracker);
   }
 
   for (const VPRecipeBase &Recipe : *BasicBlock) {
     OS << " +\n" << Indent;
     Recipe.print(OS, Indent, SlotTracker);
     OS << "\\l\"";
   }
 
   // Dump the condition bit.
   const VPValue *CBV = BasicBlock->getCondBit();
   if (CBV) {
     OS << " +\n" << Indent << " \"CondBit: ";
-    if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
-      CBI->printAsOperand(OS, SlotTracker);
+    if (const VPInstruction *CBI = CBV->getAs<VPInstruction>()) {
+      CBI->getVPValue()->printAsOperand(OS, SlotTracker);
       OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
     } else {
       CBV->printAsOperand(OS, SlotTracker);
       OS << "\"";
     }
   }
 
   bumpIndent(-2);

 }
 
 void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
                                          InterleavedAccessInfo &IAI) {
   if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
     for (VPRecipeBase &VPI : *VPBB) {
       assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions");
       auto *VPInst = cast<VPInstruction>(&VPI);
-      auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
+      auto *Inst =
+          cast<Instruction>(VPInst->getVPValue()->getUnderlyingValue());
       auto *IG = IAI.getInterleaveGroup(Inst);
       if (!IG)
         continue;
 
       auto NewIGIter = Old2New.find(IG);
       if (NewIGIter == Old2New.end())
         Old2New[IG] = new InterleaveGroup<VPInstruction>(
             IG->getFactor(), IG->isReverse(), IG->getAlign());
VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
   visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
 }
 
 void VPSlotTracker::assignSlot(const VPValue *V) {
   assert(Slots.find(V) == Slots.end() && "VPValue already has a slot!");
   const Value *UV = V->getUnderlyingValue();
   if (UV)
     return;
-  const auto *VPI = dyn_cast<VPInstruction>(V);
+  const auto *VPI = V->getAs<VPInstruction>();
   if (VPI && !VPI->hasResult())
     return;
 
   Slots[V] = NextSlot++;
 }
 
 void VPSlotTracker::assignSlots(const VPBlockBase *VPBB) {
   if (auto *Region = dyn_cast<VPRegionBlock>(VPBB))
     assignSlots(Region);
   else
     assignSlots(cast<VPBasicBlock>(VPBB));
 }
 
 void VPSlotTracker::assignSlots(const VPRegionBlock *Region) {
   ReversePostOrderTraversal<const VPBlockBase *> RPOT(Region->getEntry());
   for (const VPBlockBase *Block : RPOT)
     assignSlots(Block);
 }
 
 void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
   for (const VPRecipeBase &Recipe : *VPBB) {
     if (const auto *VPI = dyn_cast<VPInstruction>(&Recipe))
-      assignSlot(VPI);
+      assignSlot(VPI->getVPValue());
     else if (const auto *VPIV = dyn_cast<VPWidenCanonicalIVRecipe>(&Recipe))
       assignSlot(VPIV->getVPValue());
   }
 }
 
 void VPSlotTracker::assignSlots(const VPlan &Plan) {
 
   for (const VPValue *V : Plan.VPExternalDefs)
     assignSlot(V);
 
   for (auto &E : Plan.Value2VPValue)
-    if (!isa<VPInstruction>(E.second))
+    if (!E.second->isa<VPInstruction>())
       assignSlot(E.second);
 
   for (const VPValue *V : Plan.VPCBVs)
     assignSlot(V);
 
   if (Plan.BackedgeTakenCount)
     assignSlot(Plan.BackedgeTakenCount);
 
   ReversePostOrderTraversal<const VPBlockBase *> RPOT(Plan.getEntry());
   for (const VPBlockBase *Block : RPOT)
     assignSlots(Block);
 }

llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp

void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
   VPBB->setPredecessors(VPBBPreds);
 }
 
 // Add operands to VPInstructions representing phi nodes from the input IR.
 void PlainCFGBuilder::fixPhiNodes() {
   for (auto *Phi : PhisToFix) {
     assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
     VPValue *VPVal = IRDef2VPValue[Phi];
-    assert(isa<VPInstruction>(VPVal) && "Expected VPInstruction for phi node.");
-    auto *VPPhi = cast<VPInstruction>(VPVal);
+    assert(isa<VPInstruction>(VPVal->getDef()) &&
+           "Expected VPInstruction for phi node.");
+    auto *VPPhi = VPVal->getAs<VPInstruction>();
     assert(VPPhi->getNumOperands() == 0 &&
            "Expected VPInstruction with no operands.");
 
     for (Value *Op : Phi->operands())
       VPPhi->addOperand(getOrCreateVPOperand(Op));
   }
 }
 
if (auto *Br = dyn_cast<BranchInst>(Inst)) {
       continue;
     }
 
     VPInstruction *NewVPInst;
     if (auto *Phi = dyn_cast<PHINode>(Inst)) {
       // Phi node's operands may have not been visited at this point. We create
       // an empty VPInstruction that we will fix once the whole plain CFG has
       // been built.
-      NewVPInst = cast<VPInstruction>(VPIRBuilder.createNaryOp(
-          Inst->getOpcode(), {} /*No operands*/, Inst));
+      NewVPInst =
+          VPIRBuilder.createNaryOp(Inst->getOpcode(), {} /*No operands*/, Inst)
+              ->getAs<VPInstruction>();
       PhisToFix.push_back(Phi);
     } else {
       // Translate LLVM-IR operands into VPValue operands and set them in the
       // new VPInstruction.
       SmallVector<VPValue *, 4> VPOperands;
       for (Value *Op : Inst->operands())
         VPOperands.push_back(getOrCreateVPOperand(Op));
 
       // Build VPInstruction for any arbitraty Instruction without specific
       // representation in VPlan.
-      NewVPInst = cast<VPInstruction>(
-          VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
+      NewVPInst = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst)
+                      ->getAs<VPInstruction>();
     }
 
-    IRDef2VPValue[Inst] = NewVPInst;
+    IRDef2VPValue[Inst] = NewVPInst->getVPValue();
   }
 }
 
 // Main interface to build the plain CFG.
 VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
   // 1. Create the Top Region. It will be the parent of all VPBBs.
   TopRegion = new VPRegionBlock("TopRegion", false /*isReplicator*/);
 

llvm/lib/Transforms/Vectorize/VPlanSLP.cpp

VPInstruction *VPlanSlp::markFailed() {
   // FIXME: Currently this is used to signal we hit instructions we cannot
   //        trivially SLP'ize.
   CompletelySLP = false;
   return nullptr;
 }
 
 void VPlanSlp::addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New) {
   if (all_of(Operands, [](VPValue *V) {
-        return cast<VPInstruction>(V)->getUnderlyingInstr();
+        return V->getAs<VPInstruction>()->getUnderlyingInstr();
       })) {
     unsigned BundleSize = 0;
     for (VPValue *V : Operands) {
-      Type *T = cast<VPInstruction>(V)->getUnderlyingInstr()->getType();
+      Type *T = V->getAs<VPInstruction>()->getUnderlyingInstr()->getType();
       assert(!T->isVectorTy() && "Only scalar types supported for now");
       BundleSize += T->getScalarSizeInBits();
     }
     WidestBundleBits = std::max(WidestBundleBits, BundleSize);
   }
 
   auto Res = BundleToCombined.try_emplace(to_vector<4>(Operands), New);
   assert(Res.second &&
          "Already created a combined instruction for the operand bundle");
   (void)Res;
 }
 
 bool VPlanSlp::areVectorizable(ArrayRef<VPValue *> Operands) const {
   // Currently we only support VPInstructions.
   if (!all_of(Operands, [](VPValue *Op) {
-        return Op && isa<VPInstruction>(Op) &&
-               cast<VPInstruction>(Op)->getUnderlyingInstr();
+        return Op && Op->isa<VPInstruction>() &&
+               Op->getAs<VPInstruction>()->getUnderlyingInstr();
       })) {
     LLVM_DEBUG(dbgs() << "VPSLP: not all operands are VPInstructions\n");
     return false;
   }
 
   // Check if opcodes and type width agree for all instructions in the bundle.
   // FIXME: Differing widths/opcodes can be handled by inserting additional
   //        instructions.
   // FIXME: Deal with non-primitive types.
   const Instruction *OriginalInstr =
-      cast<VPInstruction>(Operands[0])->getUnderlyingInstr();
+      Operands[0]->getAs<VPInstruction>()->getUnderlyingInstr();
   unsigned Opcode = OriginalInstr->getOpcode();
   unsigned Width = OriginalInstr->getType()->getPrimitiveSizeInBits();
   if (!all_of(Operands, [Opcode, Width](VPValue *Op) {
-        const Instruction *I = cast<VPInstruction>(Op)->getUnderlyingInstr();
+        const Instruction *I = Op->getAs<VPInstruction>()->getUnderlyingInstr();
         return I->getOpcode() == Opcode &&
                I->getType()->getPrimitiveSizeInBits() == Width;
       })) {
     LLVM_DEBUG(dbgs() << "VPSLP: Opcodes do not agree \n");
     return false;
   }
 
   // For now, all operands must be defined in the same BB.
   if (any_of(Operands, [this](VPValue *Op) {
-        return cast<VPInstruction>(Op)->getParent() != &this->BB;
+        return Op->getAs<VPInstruction>()->getParent() != &this->BB;
       })) {
     LLVM_DEBUG(dbgs() << "VPSLP: operands in different BBs\n");
     return false;
   }
 
   if (any_of(Operands,
              [](VPValue *Op) { return Op->hasMoreThanOneUniqueUser(); })) {
     LLVM_DEBUG(dbgs() << "VPSLP: Some operands have multiple users.\n");
     return false;
   }
 
   // For loads, check that there are no instructions writing to memory in
   // between them.
   // TODO: we only have to forbid instructions writing to memory that could
   //       interfere with any of the loads in the bundle
   if (Opcode == Instruction::Load) {
     unsigned LoadsSeen = 0;
-    VPBasicBlock *Parent = cast<VPInstruction>(Operands[0])->getParent();
+    VPBasicBlock *Parent = Operands[0]->getAs<VPInstruction>()->getParent();
     for (auto &I : *Parent) {
       auto *VPI = cast<VPInstruction>(&I);
       if (VPI->getOpcode() == Instruction::Load &&
-          llvm::is_contained(Operands, VPI))
+          llvm::is_contained(Operands, VPI->getVPValue()))
         LoadsSeen++;
 
       if (LoadsSeen == Operands.size())
         break;
       if (LoadsSeen > 0 && VPI->mayWriteToMemory()) {
         LLVM_DEBUG(
             dbgs() << "VPSLP: instruction modifying memory between loads\n");
         return false;
       }
     }
 
     if (!all_of(Operands, [](VPValue *Op) {
-          return cast<LoadInst>(cast<VPInstruction>(Op)->getUnderlyingInstr())
+          return cast<LoadInst>(
+                     Op->getAs<VPInstruction>()->getUnderlyingInstr())
               ->isSimple();
         })) {
       LLVM_DEBUG(dbgs() << "VPSLP: only simple loads are supported.\n");
       return false;
     }
   }
 
   if (Opcode == Instruction::Store)
     if (!all_of(Operands, [](VPValue *Op) {
-          return cast<StoreInst>(cast<VPInstruction>(Op)->getUnderlyingInstr())
+          return cast<StoreInst>(
+                     Op->getAs<VPInstruction>()->getUnderlyingInstr())
               ->isSimple();
         })) {
       LLVM_DEBUG(dbgs() << "VPSLP: only simple stores are supported.\n");
       return false;
     }
 
   return true;
 }
 
 static SmallVector<VPValue *, 4> getOperands(ArrayRef<VPValue *> Values,
                                              unsigned OperandIndex) {
   SmallVector<VPValue *, 4> Operands;
   for (VPValue *V : Values) {
     // Currently we only support VPInstructions.
-    auto *U = cast<VPInstruction>(V);
+    auto *U = V->getAs<VPInstruction>();
     Operands.push_back(U->getOperand(OperandIndex));
   }
   return Operands;
 }
 
 static bool areCommutative(ArrayRef<VPValue *> Values) {
   return Instruction::isCommutative(
-      cast<VPInstruction>(Values[0])->getOpcode());
+      Values[0]->getAs<VPInstruction>()->getOpcode());
 }
 
 static SmallVector<SmallVector<VPValue *, 4>, 4>
 getOperands(ArrayRef<VPValue *> Values) {
   SmallVector<SmallVector<VPValue *, 4>, 4> Result;
-  auto *VPI = cast<VPInstruction>(Values[0]);
+  auto *VPI = Values[0]->getAs<VPInstruction>();
 
   switch (VPI->getOpcode()) {
   case Instruction::Load:
     llvm_unreachable("Loads terminate a tree, no need to get operands");
   case Instruction::Store:
     Result.push_back(getOperands(Values, 0));
     break;
   default:
     for (unsigned I = 0, NumOps = VPI->getNumOperands(); I < NumOps; ++I)
       Result.push_back(getOperands(Values, I));
     break;
   }
 
   return Result;
 }
 
 /// Returns the opcode of Values or ~0 if they do not all agree.
 static Optional<unsigned> getOpcode(ArrayRef<VPValue *> Values) {
-  unsigned Opcode = cast<VPInstruction>(Values[0])->getOpcode();
+  unsigned Opcode = Values[0]->getAs<VPInstruction>()->getOpcode();
   if (any_of(Values, [Opcode](VPValue *V) {
-        return cast<VPInstruction>(V)->getOpcode() != Opcode;
+        return V->getAs<VPInstruction>()->getOpcode() != Opcode;
       }))
     return None;
   return {Opcode};
 }
 
 /// Returns true if A and B access sequential memory if they are loads or
 /// stores or if they have identical opcodes otherwise.
 static bool areConsecutiveOrMatch(VPInstruction *A, VPInstruction *B,
static bool areConsecutiveOrMatch(VPInstruction *A, VPInstruction *B,
 
   return GA && GB && GA == GB && GA->getIndex(A) + 1 == GB->getIndex(B);
 }
 
 /// Implements getLAScore from Listing 7 in the paper.
 /// Traverses and compares operands of V1 and V2 to MaxLevel.
 static unsigned getLAScore(VPValue *V1, VPValue *V2, unsigned MaxLevel,
                            VPInterleavedAccessInfo &IAI) {
-  auto *I1 = dyn_cast<VPInstruction>(V1);
-  auto *I2 = dyn_cast<VPInstruction>(V2);
+  auto *I1 = V1->getAs<VPInstruction>();
+  auto *I2 = V2->getAs<VPInstruction>();
   // Currently we only support VPInstructions.
   if (!I1 || !I2)
     return 0;
 
   if (MaxLevel == 0)
     return (unsigned)areConsecutiveOrMatch(I1, I2, IAI);
 
   unsigned Score = 0;
VPlanSlp::getBest(OpMode Mode, VPValue *Last,
                   SmallPtrSetImpl<VPValue *> &Candidates,
                   VPInterleavedAccessInfo &IAI) {
   assert((Mode == OpMode::Load || Mode == OpMode::Opcode) &&
          "Currently we only handle load and commutative opcodes");
   LLVM_DEBUG(dbgs() << "      getBest\n");
 
   SmallVector<VPValue *, 4> BestCandidates;
   LLVM_DEBUG(dbgs() << "        Candidates  for "
-                    << *cast<VPInstruction>(Last)->getUnderlyingInstr() << " ");
+                    << *Last->getAs<VPInstruction>()->getUnderlyingInstr()
+                    << " ");
   for (auto *Candidate : Candidates) {
-    auto *LastI = cast<VPInstruction>(Last);
-    auto *CandidateI = cast<VPInstruction>(Candidate);
+    auto *LastI = Last->getAs<VPInstruction>();
+    auto *CandidateI = Candidate->getAs<VPInstruction>();
     if (areConsecutiveOrMatch(LastI, CandidateI, IAI)) {
-      LLVM_DEBUG(dbgs() << *cast<VPInstruction>(Candidate)->getUnderlyingInstr()
-                        << " ");
+      LLVM_DEBUG(dbgs() << *CandidateI->getUnderlyingInstr() << " ");
       BestCandidates.push_back(Candidate);
     }
   }
   LLVM_DEBUG(dbgs() << "\n");
 
   if (BestCandidates.empty())
     return {OpMode::Failed, nullptr};
 
for (auto *Candidate : BestCandidates) {
         BestScore = Score;
         Best = Candidate;
       }
     }
     if (!AllSame)
       break;
   }
   LLVM_DEBUG(dbgs() << "Found best "
-                    << *cast<VPInstruction>(Best)->getUnderlyingInstr()
+                    << *Best->getAs<VPInstruction>()->getUnderlyingInstr()
                     << "\n");
   Candidates.erase(Best);
 
   return {Mode, Best};
 }
 
 SmallVector<VPlanSlp::MultiNodeOpTy, 4> VPlanSlp::reorderMultiNodeOps() {
   SmallVector<MultiNodeOpTy, 4> FinalOrder;
   SmallVector<OpMode, 4> Mode;
   FinalOrder.reserve(MultiNodeOps.size());
   Mode.reserve(MultiNodeOps.size());
 
   LLVM_DEBUG(dbgs() << "Reordering multinode\n");
 
   for (auto &Operands : MultiNodeOps) {
     FinalOrder.push_back({Operands.first, {Operands.second[0]}});
-    if (cast<VPInstruction>(Operands.second[0])->getOpcode() ==
+    if (Operands.second[0]->getAs<VPInstruction>()->getOpcode() ==
         Instruction::Load)
       Mode.push_back(OpMode::Load);
     else
       Mode.push_back(OpMode::Opcode);
   }
 
   for (unsigned Lane = 1, E = MultiNodeOps[0].second.size(); Lane < E; ++Lane) {
     LLVM_DEBUG(dbgs() << "  Finding best value for lane " << Lane << "\n");
     SmallPtrSet<VPValue *, 4> Candidates;
     LLVM_DEBUG(dbgs() << "  Candidates  ");
     for (auto Ops : MultiNodeOps) {
       LLVM_DEBUG(
-          dbgs() << *cast<VPInstruction>(Ops.second[Lane])->getUnderlyingInstr()
-                 << " ");
+          dbgs()
+          << *Ops.second[Lane]->getAs<VPInstruction>()->getUnderlyingInstr()
       Candidates.insert(Ops.second[Lane]);
     }
     LLVM_DEBUG(dbgs() << "\n");
 
     for (unsigned Op = 0, E = MultiNodeOps.size(); Op < E; ++Op) {
       LLVM_DEBUG(dbgs() << "  Checking " << Op << "\n");
       if (Mode[Op] == OpMode::Failed)
         continue;
 
       VPValue *Last = FinalOrder[Op].second[Lane - 1];
       std::pair<OpMode, VPValue *> Res =
           getBest(Mode[Op], Last, Candidates, IAI);
       if (Res.second)
         FinalOrder[Op].second.push_back(Res.second);
-      else
+      else {
         // TODO: handle this case
-        FinalOrder[Op].second.push_back(markFailed());
+        markFailed();
+        FinalOrder[Op].second.push_back(nullptr);
+      }
     }
   }
 
   return FinalOrder;
 }
 
 void VPlanSlp::dumpBundle(ArrayRef<VPValue *> Values) {
   dbgs() << " Ops: ";
   for (auto Op : Values) {
-    if (auto *VPInstr = cast_or_null<VPInstruction>(Op))
+    if (auto *VPInstr = Op->getAs<VPInstruction>())
       if (auto *Instr = VPInstr->getUnderlyingInstr()) {
         dbgs() << *Instr << " | ";
         continue;
       }
     dbgs() << " nullptr | ";
   }
   dbgs() << "\n";
 }
for (auto &Operands : getOperands(Values)) {
       LLVM_DEBUG({
         dbgs() << "  Visiting Commutative";
         dumpBundle(Operands);
       });
 
       auto OperandsOpcode = getOpcode(Operands);
       if (OperandsOpcode && OperandsOpcode == getOpcode(Values)) {
         LLVM_DEBUG(dbgs() << "    Same opcode, continue building\n");
-        CombinedOperands.push_back(buildGraph(Operands));
+        CombinedOperands.push_back(buildGraph(Operands)->getVPValue());
       } else {
         LLVM_DEBUG(dbgs() << "    Adding multinode Ops\n");
         // Create dummy VPInstruction, which will we replace later by the
         // re-ordered operand.
         VPInstruction *Op = new VPInstruction(0, {});
-        CombinedOperands.push_back(Op);
+        CombinedOperands.push_back(Op->getVPValue());
         MultiNodeOps.emplace_back(Op, Operands);
       }
     }
 
     if (MultiNodeRoot) {
       LLVM_DEBUG(dbgs() << "Reorder \n");
       MultiNodeActive = false;
 
       auto FinalOrder = reorderMultiNodeOps();
 
       MultiNodeOps.clear();
       for (auto &Ops : FinalOrder) {
         VPInstruction *NewOp = buildGraph(Ops.second);
-        Ops.first->replaceAllUsesWith(NewOp);
+        VPValue *NewVal = NewOp ? NewOp->getVPValue() : nullptr;
+        Ops.first->getVPValue()->replaceAllUsesWith(NewVal);
         for (unsigned i = 0; i < CombinedOperands.size(); i++)
-          if (CombinedOperands[i] == Ops.first)
-            CombinedOperands[i] = NewOp;
+          if (CombinedOperands[i] == Ops.first->getVPValue())
+            CombinedOperands[i] = NewVal;
         delete Ops.first;
         Ops.first = NewOp;
       }
       LLVM_DEBUG(dbgs() << "Found final order\n");
     }
   } else {
     LLVM_DEBUG(dbgs() << "  NonCommuntative\n");
     if (ValuesOpcode == Instruction::Load)
       for (VPValue *V : Values)
-        CombinedOperands.push_back(cast<VPInstruction>(V)->getOperand(0));
+        CombinedOperands.push_back(
+            cast<VPInstruction>(V->getDef())->getOperand(0));
     else
-      for (auto &Operands : getOperands(Values))
-        CombinedOperands.push_back(buildGraph(Operands));
+      for (auto &Operands : getOperands(Values)) {
+        auto *Res = buildGraph(Operands);
+        CombinedOperands.push_back(Res ? Res->getVPValue() : nullptr);
+      }
   }
 
   unsigned Opcode;
   switch (ValuesOpcode) {
   case Instruction::Load:
     Opcode = VPInstruction::SLPLoad;
     break;
   case Instruction::Store:
     Opcode = VPInstruction::SLPStore;
     break;
   default:
     Opcode = ValuesOpcode;
     break;
   }
 
   if (!CompletelySLP)
     return markFailed();
 
   assert(CombinedOperands.size() > 0 && "Need more some operands");
   auto *VPI = new VPInstruction(Opcode, CombinedOperands);
-  VPI->setUnderlyingInstr(cast<VPInstruction>(Values[0])->getUnderlyingInstr());
+  VPI->setUnderlyingInstr(
+      Values[0]->getAs<VPInstruction>()->getUnderlyingInstr());
 
   LLVM_DEBUG(dbgs() << "Create VPInstruction "; VPI->print(dbgs());
-             cast<VPInstruction>(Values[0])->print(dbgs()); dbgs() << "\n");
+             Values[0]->getAs<VPInstruction>()->print(dbgs()); dbgs() << "\n");
   addCombined(Values, VPI);
   return VPI;
 }

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
       continue;
 
     VPBasicBlock *VPBB = Base->getEntryBasicBlock();
     // Introduce each ingredient into VPlan.
     for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
       VPRecipeBase *Ingredient = &*I++;
       // Can only handle VPInstructions.
       VPInstruction *VPInst = cast<VPInstruction>(Ingredient);
-      Instruction *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
+      Instruction *Inst = VPInst->getUnderlyingInstr();
       if (DeadInstructions.count(Inst)) {
-        VPInst->replaceAllUsesWith(&DummyValue);
+        VPInst->getVPValue()->replaceAllUsesWith(&DummyValue);
         Ingredient->eraseFromParent();
         continue;
       }
 
       VPRecipeBase *NewRecipe = nullptr;
       // Create VPWidenMemoryInstructionRecipe for loads and stores.
       if (LoadInst *Load = dyn_cast<LoadInst>(Inst))
         NewRecipe = new VPWidenMemoryInstructionRecipe(
for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
       } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
         NewRecipe = new VPWidenGEPRecipe(
             GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
       } else
         NewRecipe =
             new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
 
       NewRecipe->insertBefore(Ingredient);
-      VPInst->replaceAllUsesWith(&DummyValue);
+      VPInst->getVPValue()->replaceAllUsesWith(&DummyValue);
       Ingredient->eraseFromParent();
     }
   }
 }

llvm/lib/Transforms/Vectorize/VPlanValue.h

class VPValue {
   friend class VPBasicBlock;
   friend class VPInterleavedAccessInfo;
   friend class VPSlotTracker;
   friend class VPRecipeBase;
   friend class VPDef;
   friend class VPInstruction;
   friend class VPDef;
 
-  const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
-
   SmallVector<VPUser *, 1> Users;
 
 protected:
   // Hold the underlying Value, if any, attached to this VPValue.
   Value *UnderlyingVal;
 
   VPDef *Def = nullptr;
 
-  VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
+  // Set \p Val as the underlying Value of this VPValue.
+  void setUnderlyingValue(Value *Val) {
+    assert(!UnderlyingVal && "Underlying Value is already set.");
+    UnderlyingVal = Val;
+  }
 
+public:
   // DESIGN PRINCIPLE: Access to the underlying IR must be strictly limited to
   // the front-end and back-end of VPlan so that the middle-end is as
   // independent as possible of the underlying IR. We grant access to the
   // underlying IR using friendship. In that way, we should be able to use VPlan
   // for multiple underlying IRs (Polly?) by providing a new VPlan front-end,
   // back-end and analysis information for the new IR.
 
   /// Return the underlying Value attached to this VPValue.
   Value *getUnderlyingValue() { return UnderlyingVal; }
   const Value *getUnderlyingValue() const { return UnderlyingVal; }
 
-  // Set \p Val as the underlying Value of this VPValue.
-  void setUnderlyingValue(Value *Val) {
-    assert(!UnderlyingVal && "Underlying Value is already set.");
-    UnderlyingVal = Val;
-  }
-
-public:
-  /// An enumeration for keeping track of the concrete subclass of VPValue that
-  /// are actually instantiated. Values of this enumeration are kept in the
-  /// SubclassID field of the VPValue objects. They are used for concrete
-  /// type identification.
-  enum {
-    VPValueSC,
-    VPInstructionSC,
-    VPMemoryInstructionSC,
-    VPVWidenCallSC,
-    VPVWidenSelectSC,
-    VPVWidenGEPSC
-  };
-
-  VPValue(Value *UV = nullptr, VPDef *Def = nullptr)
-      : VPValue(VPValueSC, UV, Def) {}
+  VPValue(Value *UV = nullptr, VPDef *Def = nullptr);
   VPValue(const VPValue &) = delete;
   VPValue &operator=(const VPValue &) = delete;
 
   virtual ~VPValue() {
+    assert(!Def && "trying to delete a VPVAlue with a remaining linked def");
     assert(Users.empty() && "trying to delete a VPValue with remaining users");
   }
 
-  /// \return an ID for the concrete type of this object.
-  /// This is used to implement the classof checks. This should not be used
-  /// for any other purpose, as the values may change as LLVM evolves.
-  unsigned getVPValueID() const { return SubclassID; }
-
   void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const;
   void print(raw_ostream &OS, VPSlotTracker &Tracker) const;
 
   /// Dump the value to stderr (for debugging).
   void dump() const;
 
   unsigned getNumUsers() const { return Users.size(); }
   void addUser(VPUser &User) { Users.push_back(&User); }
bool hasMoreThanOneUniqueUser() {
     while (Current != user_end() && *user_begin() == *Current)
       Current++;
     return Current != user_end();
   }
 
   void replaceAllUsesWith(VPValue *New);
 
   VPDef *getDef() { return Def; }
+  const VPDef *getDef() const { return Def; }
+
+  template <class DestTy> DestTy *getAs() {
+    return dyn_cast_or_null<DestTy>(getDef());
+  }
+  template <class DestTy> const DestTy *getAs() const {
+    return dyn_cast_or_null<DestTy>(getDef());
+  }
+
+  template <class DestTy> bool isa() const {
+    return Def && llvm::isa<DestTy>(Def);
+  }
 };
 
 typedef DenseMap<Value *, VPValue *> Value2VPValueTy;
 typedef DenseMap<VPValue *, Value *> VPValue2ValueTy;
 
 raw_ostream &operator<<(raw_ostream &OS, const VPValue &V);
 
 /// This class augments VPValue with operands which provide the inverse def-use
public:
   }
 
   unsigned getNumOperands() const { return Operands.size(); }
   inline VPValue *getOperand(unsigned N) const {
     assert(N < Operands.size() && "Operand index out of bounds");
     return Operands[N];
   }
 
+  inline VPDef *getOperandAsDef(unsigned N) { return getOperand(N)->getDef(); }
+
   void setOperand(unsigned I, VPValue *New) {
     Operands[I]->removeUser(*this);
     Operands[I] = New;
     New->addUser(*this);
   }
 
   typedef SmallVectorImpl<VPValue *>::iterator operand_iterator;
   typedef SmallVectorImpl<VPValue *>::const_iterator const_operand_iterator;

llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp

TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
 
   auto Iter = VecBB->begin();
   VPInstruction *Phi = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::PHI, Phi->getOpcode());
 
   VPInstruction *Idx = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::GetElementPtr, Idx->getOpcode());
   EXPECT_EQ(2u, Idx->getNumOperands());
-  EXPECT_EQ(Phi, Idx->getOperand(1));
+  EXPECT_EQ(Phi->getVPValue(), Idx->getOperand(1));
 
   VPInstruction *Load = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::Load, Load->getOpcode());
   EXPECT_EQ(1u, Load->getNumOperands());
-  EXPECT_EQ(Idx, Load->getOperand(0));
+  EXPECT_EQ(Idx->getVPValue(), Load->getOperand(0));
 
   VPInstruction *Add = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::Add, Add->getOpcode());
   EXPECT_EQ(2u, Add->getNumOperands());
-  EXPECT_EQ(Load, Add->getOperand(0));
+  EXPECT_EQ(Load->getVPValue(), Add->getOperand(0));
 
   VPInstruction *Store = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::Store, Store->getOpcode());
   EXPECT_EQ(2u, Store->getNumOperands());
-  EXPECT_EQ(Add, Store->getOperand(0));
-  EXPECT_EQ(Idx, Store->getOperand(1));
+  EXPECT_EQ(Add->getVPValue(), Store->getOperand(0));
+  EXPECT_EQ(Idx->getVPValue(), Store->getOperand(1));
 
   VPInstruction *IndvarAdd = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::Add, IndvarAdd->getOpcode());
   EXPECT_EQ(2u, IndvarAdd->getNumOperands());
-  EXPECT_EQ(Phi, IndvarAdd->getOperand(0));
+  EXPECT_EQ(Phi->getVPValue(), IndvarAdd->getOperand(0));
 
   VPInstruction *ICmp = dyn_cast<VPInstruction>(&*Iter++);
   EXPECT_EQ(Instruction::ICmp, ICmp->getOpcode());
   EXPECT_EQ(2u, ICmp->getNumOperands());
-  EXPECT_EQ(IndvarAdd, ICmp->getOperand(0));
-  EXPECT_EQ(VecBB->getCondBit(), ICmp);
+  EXPECT_EQ(IndvarAdd->getVPValue(), ICmp->getOperand(0));
+  EXPECT_EQ(VecBB->getCondBit(), ICmp->getVPValue());
 
   // Add an external value to check we do not print the list of external values,
   // as this is not required with the new printing.
   Plan->addVPValue(&*F->arg_begin());
   std::string FullDump;
   raw_string_ostream(FullDump) << *Plan;
   const char *ExpectedStr = R"(digraph VPlan {
 graph [labelloc=t, fontsize=30; label="Vectorization Plan"]

llvm/unittests/Transforms/Vectorize/VPlanPredicatorTest.cpp

TEST_F(VPlanPredicatorTest, BasicPredicatorTest) {
   // Test block predicates
   EXPECT_NE(nullptr, CBV1);
   EXPECT_NE(nullptr, CBV2);
   EXPECT_NE(nullptr, InnerAnd);
   EXPECT_EQ(CBV1, OuterIfPred);
   EXPECT_EQ(InnerAnd->getOpcode(), Instruction::And);
   EXPECT_EQ(InnerAnd->getOperand(0), CBV1);
   EXPECT_EQ(InnerAnd->getOperand(1), CBV2);
-  EXPECT_EQ(InnerIfPred, InnerAnd);
+  EXPECT_EQ(InnerIfPred, InnerAnd->getVPValue());
 
   // Test Linearization
   EXPECT_EQ(InnerLoopLinSucc, OuterIf);
   EXPECT_EQ(OuterIfLinSucc, InnerIf);
   EXPECT_EQ(InnerIfLinSucc, InnerLoopLatch);
 
   // Check that the containing VPlan is set correctly.
   EXPECT_EQ(&*Plan, InnerLoopLinSucc->getPlan());
TEST_F(VPlanPredicatorTest, PredicatorNegOrTest) {
   EXPECT_EQ(Not->getOpcode(), VPInstruction::Not);
 
   EXPECT_EQ(And->getOperand(0), OuterCBV);
   EXPECT_EQ(And->getOperand(1), InnerCBV);
 
   EXPECT_EQ(Not->getOperand(0), InnerCBV);
 
   EXPECT_EQ(NotAnd->getOperand(0), OuterCBV);
-  EXPECT_EQ(NotAnd->getOperand(1), Not);
+  EXPECT_EQ(NotAnd->getOperand(1), Not->getVPValue());
 
-  EXPECT_EQ(InnerIfTSucc->getPredicate(), And);
-  EXPECT_EQ(InnerIfFSucc->getPredicate(), NotAnd);
+  EXPECT_EQ(InnerIfTSucc->getPredicate(), And->getVPValue());
+  EXPECT_EQ(InnerIfFSucc->getPredicate(), NotAnd->getVPValue());
 
   EXPECT_EQ(TSuccSucc, FSuccSucc);
   EXPECT_EQ(Or->getOpcode(), Instruction::Or);
-  EXPECT_EQ(TSuccSucc->getPredicate(), Or);
+  EXPECT_EQ(TSuccSucc->getPredicate(), Or->getVPValue());
 
   // Test operands of the Or - account for differences in predecessor block
   // ordering.
-  VPInstruction *OrOp0Inst = cast<VPInstruction>(Or->getOperand(0));
-  VPInstruction *OrOp1Inst = cast<VPInstruction>(Or->getOperand(1));
+  VPInstruction *OrOp0Inst = cast<VPInstruction>(Or->getOperandAsDef(0));
+  VPInstruction *OrOp1Inst = cast<VPInstruction>(Or->getOperandAsDef(1));
 
   bool ValidOrOperands = false;
   if (((OrOp0Inst == And) && (OrOp1Inst == NotAnd)) ||
       ((OrOp0Inst == NotAnd) && (OrOp1Inst == And)))
     ValidOrOperands = true;
 
   EXPECT_TRUE(ValidOrOperands);
 }
 
 } // namespace
 } // namespace llvm

llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp

TEST_F(VPlanSlpTest, testSlpSimple_2) {
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
   EXPECT_EQ(64u, Slp.getWidestBundleBits());
   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 
-  auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
+  auto *CombinedAdd =
+      cast<VPInstruction>(CombinedStore->getOperand(0)->getDef());
   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 
-  auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
-  auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
+  auto *CombinedLoadA =
+      cast<VPInstruction>(CombinedAdd->getOperand(0)->getDef());
+  auto *CombinedLoadB =
+      cast<VPInstruction>(CombinedAdd->getOperand(1)->getDef());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 
   delete CombinedStore;
   delete CombinedAdd;
   delete CombinedLoadA;
   delete CombinedLoadB;
 }
TEST_F(VPlanSlpTest, testSlpSimple_3) {
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 
   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
   EXPECT_EQ(64u, Slp.getWidestBundleBits());
   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 
-  auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
+  auto *CombinedAdd =
+      cast<VPInstruction>(CombinedStore->getOperand(0)->getDef());
   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 
-  auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
-  auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
+  auto *CombinedLoadA =
+      cast<VPInstruction>(CombinedAdd->getOperand(0)->getDef());
+  auto *CombinedLoadB =
+      cast<VPInstruction>(CombinedAdd->getOperand(1)->getDef());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 
   VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1));
   VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3));
-  EXPECT_EQ(GetA, CombinedLoadA->getOperand(0));
-  EXPECT_EQ(GetB, CombinedLoadB->getOperand(0));
+  EXPECT_EQ(GetA->getVPValue(), CombinedLoadA->getOperand(0));
+  EXPECT_EQ(GetB->getVPValue(), CombinedLoadB->getOperand(0));
 
   delete CombinedStore;
   delete CombinedAdd;
   delete CombinedLoadA;
   delete CombinedLoadB;
 }
 
 TEST_F(VPlanSlpTest, testSlpReuse_1) {
TEST_F(VPlanSlpTest, testSlpReuse_1) {
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
 
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
   EXPECT_EQ(64u, Slp.getWidestBundleBits());
   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 
-  auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
+  auto *CombinedAdd =
+      cast<VPInstruction>(CombinedStore->getOperand(0)->getDef());
   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 
-  auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
-  EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1));
+  auto *CombinedLoadA =
+      cast<VPInstruction>(CombinedAdd->getOperand(0)->getDef());
+  EXPECT_EQ(CombinedLoadA->getVPValue(), CombinedAdd->getOperand(1));
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 
   delete CombinedStore;
   delete CombinedAdd;
   delete CombinedLoadA;
 }
 
 TEST_F(VPlanSlpTest, testSlpReuse_2) {
TEST_F(VPlanSlpTest, testSlpReuse_2) {
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
 
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   Slp.buildGraph(StoreRoot);
   EXPECT_FALSE(Slp.isCompletelySLP());
 }
 
 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2,
                                 VPBasicBlock *Body,
                                 VPInterleavedAccessInfo &&IAI) {
   VPlanSlp Slp(IAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 
   EXPECT_TRUE(Slp.isCompletelySLP());
   EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore);
 
   VPInstruction *CombinedAdd =
-      cast<VPInstruction>(CombinedStore->getOperand(0));
+      cast<VPInstruction>(CombinedStore->getOperand(0)->getDef());
   EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add);
 
   VPInstruction *CombinedMulAB =
-      cast<VPInstruction>(CombinedAdd->getOperand(0));
+      cast<VPInstruction>(CombinedAdd->getOperand(0)->getDef());
   VPInstruction *CombinedMulCD =
-      cast<VPInstruction>(CombinedAdd->getOperand(1));
+      cast<VPInstruction>(CombinedAdd->getOperand(1)->getDef());
   EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul);
 
   VPInstruction *CombinedLoadA =
-      cast<VPInstruction>(CombinedMulAB->getOperand(0));
+      cast<VPInstruction>(CombinedMulAB->getOperand(0)->getDef());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
   VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2));
   VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0));
   EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1));
 
   VPInstruction *CombinedLoadB =
-      cast<VPInstruction>(CombinedMulAB->getOperand(1));
+      cast<VPInstruction>(CombinedMulAB->getOperand(1)->getDef());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
   VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4));
   VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
   EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0));
   EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1));
 
   EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul);
 
   VPInstruction *CombinedLoadC =
-      cast<VPInstruction>(CombinedMulCD->getOperand(0));
+      cast<VPInstruction>(CombinedMulCD->getOperand(0)->getDef());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode());
   VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7));
   VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17));
   EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0));
   EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1));
 
   VPInstruction *CombinedLoadD =
-      cast<VPInstruction>(CombinedMulCD->getOperand(1));
+      cast<VPInstruction>(CombinedMulCD->getOperand(1)->getDef());
   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode());
   VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9));
   VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19));
   EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0));
   EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1));
 
   delete CombinedStore;
   delete CombinedAdd;
TEST_F(VPlanSlpTest, testSlpReorder_3) {
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 
   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 
   // FIXME Need to select better first value for lane0.
   EXPECT_FALSE(Slp.isCompletelySLP());
 }
 
 TEST_F(VPlanSlpTest, testSlpReorder_4) {
   LLVMContext Ctx;
TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) {
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5));
 
   VPlanSlp Slp(VPIAI, *BB2);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
   EXPECT_EQ(0u, Slp.getWidestBundleBits());
 }
 
 // Make sure we do not combine instructions with operands in different BBs.
 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
   const char *ModuleString =
       "%struct.Test = type { i32, i32 }\n"
TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
 
   VPlanSlp Slp(VPIAI, *BB2);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
   EXPECT_EQ(0u, Slp.getWidestBundleBits());
 }
 
 TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
   const char *ModuleString =
       "%struct.Test = type { i32, i32 }\n"
       "%struct.Test3 = type { i32, i32, i32 }\n"
TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
   EXPECT_FALSE(Slp.isCompletelySLP());
 }
 
 TEST_F(VPlanSlpTest, testSlpAtomicStore) {
   const char *ModuleString =
       "%struct.Test = type { i32, i32 }\n"
       "%struct.Test3 = type { i32, i32, i32 }\n"
TEST_F(VPlanSlpTest, testSlpAtomicStore) {
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 
   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 
   VPlanSlp Slp(VPIAI, *Body);
-  SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
+  SmallVector<VPValue *, 4> StoreRoot = {Store1->getVPValue(),
+                                         Store2->getVPValue()};
   Slp.buildGraph(StoreRoot);
   EXPECT_FALSE(Slp.isCompletelySLP());
 }
 
 } // namespace
 } // namespace llvm

llvm/unittests/Transforms/Vectorize/VPlanTest.cpp

EXPECT_EQ(ExpectedStr, FullDump);
     I3->print(OS);
     OS.flush();
     EXPECT_EQ("br vp<%0> vp<%1>", I3Dump);
   }
 
   {
     std::string I4Dump;
     raw_string_ostream OS(I4Dump);
-    OS << *I4;
+    OS << *I4->getVPValue();
     OS.flush();
     EXPECT_EQ("vp<%2> = mul vp<%1> vp<%0>", I4Dump);
   }
 }
 
 TEST(VPRecipeTest, CastVPInstructionToVPUser) {
   VPValue Op1;
   VPValue Op2;