diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -378,7 +378,7 @@ /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph. /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock. -class VPBlockBase { +class VPBlockBase : public VPUser { friend class VPBlockUtils; const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). @@ -396,9 +396,6 @@ /// List of successor blocks. SmallVector Successors; - /// Successor selector, null for zero or single successor blocks. - VPValue *CondBit = nullptr; - /// Current block predicate - null if the block does not need a predicate. VPValue *Predicate = nullptr; @@ -434,7 +431,7 @@ protected: VPBlockBase(const unsigned char SC, const std::string &N) - : SubclassID(SC), Name(N) {} + : VPUser(), SubclassID(SC), Name(N) {} public: /// An enumeration for keeping track of the concrete subclass of VPBlockBase @@ -502,6 +499,8 @@ size_t getNumSuccessors() const { return Successors.size(); } size_t getNumPredecessors() const { return Predecessors.size(); } + void dump() const; + /// An Enclosing Block of a block B is any block containing B, including B /// itself. \return the closest enclosing block starting from "this", which /// has successors. \return the root enclosing block if all enclosing blocks @@ -546,11 +545,29 @@ } /// \return the condition bit selecting the successor. - VPValue *getCondBit() { return CondBit; } + VPValue *getCondBit() { + if (getNumOperands()) + return getOperand(0); + return nullptr; + } - const VPValue *getCondBit() const { return CondBit; } + const VPValue *getCondBit() const { + if (getNumOperands()) + return getOperand(0); + return nullptr; + } - void setCondBit(VPValue *CV) { CondBit = CV; } + void setCondBit(VPValue *CV) { + if (!CV) { + if (getNumOperands() == 1) + removeLastOperand(); + return; + } + if (getNumOperands() == 1) + setOperand(0, CV); + else + addOperand(CV); + } VPValue *getPredicate() { return Predicate; } @@ -574,7 +591,7 @@ VPValue *Condition) { assert(Successors.empty() && "Setting two successors when others exist."); assert(Condition && "Setting two successors without condition!"); - CondBit = Condition; + setCondBit(Condition); appendSuccessor(IfTrue); appendSuccessor(IfFalse); } @@ -594,7 +611,7 @@ /// Remove all the successors of this block and set to null its condition bit void clearSuccessors() { Successors.clear(); - CondBit = nullptr; + setCondBit(nullptr); } /// The method which generates the output IR that correspond to this @@ -1732,9 +1749,6 @@ /// Holds the VPLoopInfo analysis for this VPlan. VPLoopInfo VPLInfo; - /// Holds the condition bit values built during VPInstruction to VPRecipe transformation. - SmallVector VPCBVs; - public: VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) { if (Entry) @@ -1755,8 +1769,6 @@ delete BackedgeTakenCount; for (VPValue *Def : VPExternalDefs) delete Def; - for (VPValue *CBV : VPCBVs) - delete CBV; } /// Generate the IR code for this VPlan. @@ -1792,11 +1804,6 @@ VPExternalDefs.insert(VPVal); } - /// Add \p CBV to the vector of condition bit values. - void addCBV(VPValue *CBV) { - VPCBVs.push_back(CBV); - } - void addVPValue(Value *V) { assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1134,9 +1134,6 @@ for (const VPValue *V : Plan.VPExternalDefs) assignSlot(V); - for (const VPValue *V : Plan.VPCBVs) - assignSlot(V); - if (Plan.BackedgeTakenCount) assignSlot(Plan.BackedgeTakenCount); diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -253,7 +253,13 @@ assert((PreheaderBB->getTerminator()->getNumSuccessors() == 1) && "Unexpected loop preheader"); VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(PreheaderBB); - createVPInstructionsForVPBB(PreheaderVPBB, PreheaderBB); + for (auto &I : *PreheaderBB) { + if (I.getType()->isVoidTy()) + continue; + VPValue *VPV = new VPValue(&I); + Plan.addExternalDef(VPV); + IRDef2VPValue[&I] = VPV; + } // Create empty VPBB for Loop H so that we can link PH->H. VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader()); // Preheader's predecessors will be set during the loop RPO traversal below. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -24,17 +24,6 @@ auto *TopRegion = cast(Plan->getEntry()); ReversePostOrderTraversal RPOT(TopRegion->getEntry()); - // Condition bit VPValues get deleted during transformation to VPRecipes. - // Create new VPValues and save away as condition bits. These will be deleted - // after finalizing the vector IR basic blocks. - for (VPBlockBase *Base : RPOT) { - VPBasicBlock *VPBB = Base->getEntryBasicBlock(); - if (auto *CondBit = VPBB->getCondBit()) { - auto *NCondBit = new VPValue(CondBit->getUnderlyingValue()); - VPBB->setCondBit(NCondBit); - Plan->addCBV(NCondBit); - } - } for (VPBlockBase *Base : RPOT) { // Do not widen instructions in pre-header and exit blocks. if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0) diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -233,6 +233,11 @@ New->addUser(*this); } + void removeLastOperand() { + VPValue *Op = Operands.pop_back_val(); + Op->removeUser(*this); + } + typedef SmallVectorImpl::iterator operand_iterator; typedef SmallVectorImpl::const_iterator const_operand_iterator; typedef iterator_range operand_range;